{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T13:13:51.560776Z",
     "start_time": "2018-08-04T13:13:42.851900Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:11:21.306294Z",
     "start_time": "2018-08-04T14:11:20.816583Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/LB804_train_pred.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:11:30.377491Z",
     "start_time": "2018-08-04T14:11:30.366702Z"
    }
   },
   "outputs": [],
   "source": [
    "def predictionDist(y, yhat):\n",
    "    sub = y.to_frame('y')\n",
    "    sub['yhat'] = yhat\n",
    "    y0 = sub[sub.y==0]\n",
    "    y1 = sub[sub.y==1]\n",
    "    y0.yhat.name = 'Actual:0'\n",
    "    y0.yhat.plot(kind='kde',legend=True)\n",
    "    y1.yhat.name = 'Actual:1'\n",
    "    y1.yhat.plot(kind='kde',legend=True)\n",
    "    plt.xlim(0,1)\n",
    "    plt.xlabel('Probability of Event')\n",
    "    plt.title('Prediction Distribution', fontsize=13, alpha=0.5)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:11:30.527611Z",
     "start_time": "2018-08-04T14:11:30.506424Z"
    }
   },
   "outputs": [],
   "source": [
    "def feature_vs_y(feature, y, feature_name=None, kind='kde', bw_method=None, bins=100):\n",
    "    \n",
    "    if feature_name is None:\n",
    "        feature_name = feature.name\n",
    "    \n",
    "    df = y.to_frame('y')\n",
    "    df[feature_name] = feature\n",
    "    y0 = df[df.y==0]\n",
    "    y1 = df[df.y==1]\n",
    "    y0[feature_name].name = 'Actual:0'\n",
    "    y1[feature_name].name = 'Actual:1'\n",
    "    if kind=='kde':\n",
    "        y0[feature_name].plot(kind='kde', legend=True, bw_method=bw_method)\n",
    "        y1[feature_name].plot(kind='kde', legend=True, bw_method=bw_method)\n",
    "    elif kind=='hist':\n",
    "        y0[feature_name].plot(kind='hist',legend=True, bins=bins)\n",
    "        y1[feature_name].plot(kind='hist',legend=True, bins=bins)\n",
    "        \n",
    "    plt.xlabel(f'{feature_name}')\n",
    "    plt.title(f'{feature_name} Distribution', fontsize=13, alpha=0.5)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T03:10:49.568633Z",
     "start_time": "2018-08-04T03:10:49.136480Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEXCAYAAABlI9noAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHilJREFUeJzt3X2YVXW99/H3R8CQQmWUOsZoQx1MTUtzfMjsHJVUtFKPWlGW6OUdd912l1enB/LO1FIv63iyPD14PMIROQWZ3iInNXyW6lZwUFMQvSClGPQkAqKEyNP3/mP9hrMZ98ysgbX2nj3zeV3Xvmat3/qttb9rAfPl97B/WxGBmZlZEXaqdwBmZtZ/OKmYmVlhnFTMzKwwTipmZlYYJxUzMyuMk4qZmRXGScWslyTtKukSSbvv4HXeK+kLRcWVrvkOSZMq9k+TdEqR75Gu+xlJHyz6utb4Btc7ALP+RtLBwKnARiCATcCLwJPAY5E+HBYRTwBP5LzmBcB96ZwuRcSfgCu3P/qq730JMCUi/lzxPv9R5HtY/+GkYgOWpEERsbmky6+OiGvS++wMvAs4CRgD/LKMNyz5fsxycVKxPkXSYUBrRPysoqwJ+CJwTUS83M25pwGDyFoH7wbWAQ9GxOPp+MHA3wHzgSOA14GfSBoGHE/2i38w8BxwZ0SsTee9BfgY0AKsBX7fm3uKiA3AIknrgHMlvSsi/tgRT0XyORA4BtiVrJWzOCJmSvo0sBtwiqSPAssiYpqkc4D/AnYHRgO/ldQOnB0R36kIYYik07t7Jh0xVDzHLRExq6J77rOSAliQys8Bno2IOemctwHjgL2A14DHgN9FxJbUTXgBcCtwdLqXZcDMiHi1N8/S+j4nFetrngCOlzQqIpanskPIfoF1mVAqvAeYmV6jgU9JWhkRy9Lx3YHhwL8ASBIwHngJ+CmwGTgZOAOYms45gywBXU32b+YT23NjEfEnSa+muP5YeUzSEOB0YFpEPJdaN3ul837RTffXIcAMstbPYGBUlbfu6Zl0F/PPUvfXtMrur06xDwXOBuYBPwdGAJ8me5aVCfg9wL+n8s8AxwKzeorBGosH6q1PiYjXgQVkvyyRtBNwMPBozku0R8QTEbElIv4IPJXO77AFuCciNkbERrJf3HsBd0TE+lR2NzA6DcjvSvaL+K50fC3w4A7c4ivAsC6ObQb2lLRLRGxI4yM9eSoinovMxi7q9PRMdtQYstjnRMSmiFhBlkze36negxGxLv0ZPwm8vcAYrI9wS8X6ovnA2ZJmA+8k+8/PMznP7dyaeZn0P/7k1YjYVLE/guzfwVezRstWm8i6aTpWXK287uqcsVSzK1n32jYiYqOknwMfAMZKWg38v4h4sofr5Wm99fRMdtRuwMsdExCSVWT3Wqmyq2sDsHOBMVgf4aRifU5ELJe0iqy7ZD/g8V4MQHee5rs7Wetg6+U7HX+Z7Bfc9zr9UgSy6cMV11mVtkfkjKXztfYh63p7Q1IBiIilwNLUOns38AlJyyNiVZW4t56W4627eybVfrkPB9b04j3WALtJUsUzHMG2z90GCHd/WV81n+x/7WPI3/UF0CzpIEk7SRoNHAD8oZv6z5MNdp+UBuyR9OY0aE5EvAIsJRvneVMatP+73tyIpJ0l7Uc2NvN06oLqXOctkg6QNDQitgDr06Et6edaYI/evG+F7p7JfwFvlrSvMvsD7+h0fk/vvZjsP6gfkjRI0p5kA/K9+XOzfsItFeurngROIJvptLIX5y0kS0QfJZuFdHtXA8wAERGSZgDHARNTYvkr2UD6glTtFrLZX1/hv2d/df7F29kISRem7Y7PqfyWrn/RCjiMbIbXTmT/+59ZMTlhDnCypCPIxkh68zmRLp9JRKySdGe6vyGp7qJO598HHCvpRGBhRPxn5cGIWC9pGtnsr6PIEuLjwEO9iNH6CflLuqwvSrOyvgzcm2NcoeOcrVNhSw3OzLrk7i/rqw4i+8zJU/UOxMzyc/eX9TmSvk42lnBb5QC9pPeSdeFU8+taxGZm3XP3l5mZFcbdX2ZmVpgB1/215557RktLS73DMDNrGPPnz38pIkbmqTvgkkpLSwttbW31DsPMrGFIyrNkEODuLzMzK5CTipmZFcZJxczMCjPgxlTMbODYuHEj7e3trF+/vufKxtChQ2lubmbIkCHbfQ0nFTPrt9rb2xk+fDgtLS10+moD6yQiWLlyJe3t7YwePXq7r+PuLzPrt9avX88ee+zhhJKDJPbYY48dbtU5qZhZv+aEkl8Rz8pJxczMCuMxFTMbMFom3V7o9ZZe+ZFc9W699VZOP/10Fi1axH777ddlvRtuuIETTjiBt7/97dsVzwMPPMBVV13Fr3/9xvVVp06dymWXXQbAt771LSZMmLBd79ETt1S2U8uk27e+zMy6M336dI4++mhmzJjRbb0bbriB559/vvD3X7VqFZdeeilz585l3rx5XHrppaxevbrw9wEnFTOzUq1du5bf//73TJ48eZuk8v3vf5+DDjqI973vfUyaNImbb76ZtrY2zjrrLA4++GBee+01WlpaeOmllwBoa2vjmGOOAWDevHkcddRRHHLIIRx11FE888wz3cYwe/Zsjj/+eJqamhgxYgTHH388v/nNb0q5X3d/mZmVaObMmYwbN459992XpqYmHn30Uf7yl78wc+ZM5s6dy7Bhw1i1ahVNTU38+Mc/5qqrrqK1tbXba+63337MmTOHwYMHc88993DhhRdyyy23bFOnra2Na6+9luuvv57ly5ez9957bz3W3NzM8uXLS7lfJxUzsxJNnz6dCy64AIDx48czffp0tmzZwrnnnsuwYcMAaGpq6tU116xZw4QJE1i8eDGS2Lhx4xvqtLa2cv311wPZZ1A6K2tWnJOKmVlJVq5cyX333ceCBQuQxObNm5HEGWeckeuX+uDBg9myZQvANp8fueiiizj22GO59dZbWbp06dZusa40NzfzwAMPbN1vb2/v8Zzt5TEVM7OS3HzzzZx99tn86U9/YunSpSxbtozRo0fT1NTElClTWLduHZANpAMMHz6cV199dev5LS0tzJ8/H2Cb7q01a9YwatQoIBvc78mJJ57IXXfdxerVq1m9ejV33XUXJ554YlG3uQ23VMxswMg7Bbgo06dPZ9KkSduUnXHGGSxatIhTTjmF1tZWdt55Z04++WSuuOIKzjnnHD7/+c+zyy678NBDD3HxxRdz3nnnccUVV3DEEUdsvcbXv/51JkyYwA9+8AOOO+64qu9dOabS1NTERRddxGGHHQbAt7/97V53ueU14L6jvrW1NYr4kq7KqcS1/otqZvksWrSI/fffv95hNJRqz0zS/IjofvZA4u4vMzMrjJOKmZkVxknFzMwK46RiZmaFcVIxM7PCOKmYmVlh/DkVMxs4Ltmt4OutyVWtLyx9P27cOB5++GGOPvroqseL4paKmVnJ6r30PcDXvvY1pk2bVsq1KzmpmJmVqC8sfQ8wduxYhg8fXso9VnL3l5lZifrC0ve15KRiZlaivrD0fS05qZiZlaSvLH1fSx5TMTMrSV9Z+r6W3FIxs4Ej5xTgovSVpe8BPvShD/H000+zdu1ampubmTx5cinfqVL60veSBgFtwPKI+Kik0cAMoAl4FPhsRGyQ9CbgRuBQYCXwyYhYmq7xTeA8YDPwpYiYncrHAT8CBgHXR8SVPcXjpe/NBg4vfd97jbD0/ZeBRRX73wOujogxwGqyZEH6uToi/ha4OtVD0gHAeOA9wDjgp5IGpWT1E+Ak4ADgU6mumZnVSalJRVIz8BHg+rQv4Djg5lRlKnBa2j417ZOOj031TwVmRMTrEfEcsAQ4PL2WRMSzEbGBrPVzapn3Y2Zm3Su7pfJD4OvAlrS/B/ByRGxK++3AqLQ9ClgGkI6vSfW3lnc6p6vyN5A0UVKbpLYVK1bs6D2ZWQMZaN9uuyOKeFalJRVJHwVejIj5lcVVqkYPx3pb/sbCiOsiojUiWkeOHNlN1GbWnwwdOpSVK1c6seQQEaxcuZKhQ4fu0HXKnP31QeAUSScDQ4FdyVouu0sanFojzUDHQjftwN5Au6TBwG7AqoryDpXndFVuZkZzczPt7e24hyKfoUOH0tzcvEPXKC2pRMQ3gW8CSDoG+GpEnCXpV8CZZGMgE4Db0imz0v5D6fh9ERGSZgG/kPQD4O3AGGAeWUtlTJpNtpxsMP/TZd2PmTWeIUOGMHr06HqHMaDU43Mq3wBmSLoMeAyYnMonA9MkLSFroYwHiIiFkm4CngI2AedHxGYASV8EZpNNKZ4SEQtreidmZraNmiSViHgAeCBtP0s2c6tznfXAx7s4/3Lg8irldwB3FBiqmZntAC/TYmZmhXFSMTOzwjipmJlZYZxUzMysME4qZmZWGCcVMzMrjJOKmZkVxl/SVQB/t4qZWcYtFTMzK4yTipmZFcZJxczMCuOkYmZmhXFSMTOzwjipmJlZYZxUzMysME4qZmZWGCcVMzMrjJOKmZkVxknFzMwK46RiZmaFcVIxM7PCOKmYmVlhnFTMzKwwTipmZlYYJxUzMyuMk4qZmRXGScXMzArjpGJmZoVxUjEzs8I4qZiZWWGcVMzMrDBOKmZmVhgnFTMzK4yTipmZFcZJxczMCuOkYmZmhSktqUgaKmmepD9IWijp0lQ+WtJcSYsl/VLSzqn8TWl/STreUnGtb6byZySdWFE+LpUtkTSprHsxM7N8ymypvA4cFxHvAw4Gxkk6EvgecHVEjAFWA+el+ucBqyPib4GrUz0kHQCMB94DjAN+KmmQpEHAT4CTgAOAT6W6ZmZWJ6UllcisTbtD0iuA44CbU/lU4LS0fWraJx0fK0mpfEZEvB4RzwFLgMPTa0lEPBsRG4AZqa6ZmdVJqWMqqUXxOPAicDfwR+DliNiUqrQDo9L2KGAZQDq+BtijsrzTOV2VV4tjoqQ2SW0rVqwo4tbMzKyKUpNKRGyOiIOBZrKWxf7VqqWf6uJYb8urxXFdRLRGROvIkSN7DtzMzLZLTWZ/RcTLwAPAkcDukganQ83A82m7HdgbIB3fDVhVWd7pnK7KzcysTsqc/TVS0u5pexfgw8Ai4H7gzFRtAnBb2p6V9knH74uISOXj0+yw0cAYYB7wCDAmzSbbmWwwf1ZZ92NmZj0b3HMVkHRgRCzo5bX3AqamWVo7ATdFxK8lPQXMkHQZ8BgwOdWfDEyTtISshTIeICIWSroJeArYBJwfEZtTXF8EZgODgCkRsbCXMZqZWYFyJRXg2tQauAH4RerO6lZEPAEcUqX8WbLxlc7l64GPd3Gty4HLq5TfAdzRUyxmZlYbubq/IuJo4CyyMYw2Sb+QdHypkZmZWcPJPaYSEYuBbwHfAP4euEbS05JOLys4MzNrLLmSiqT3SrqabKD9OOBjEbF/2r66xPjMzKyB5B1T+THwb8CFEfFaR2FEPC/pW6VE1qBaJt0OwNIrP1LnSMzMai9vUjkZeK1i1tVOwNCIWBcR00qLzszMGkreMZV7gF0q9oelMjMzs63yJpWhFYtDkraHlROSmZk1qrxJ5a+S3t+xI+lQ4LVu6puZ2QCUd0zlAuBXkjrW1toL+GQ5IZmZWaPKlVQi4hFJ+wHvJlsd+OmI2FhqZGZm1nDytlQADgNa0jmHSCIibiwlKjMza0h5F5ScBrwLeBzYnIoDcFIxM7Ot8rZUWoED0lL0ZmZmVeWd/bUA+JsyAzEzs8aXt6WyJ/CUpHnA6x2FEXFKKVGZmVlDyptULikzCDMz6x/yTil+UNI7gDERcY+kYWTftmhmZrZV3tlfnwMmAk1ks8BGAdcCY8sLre/pWIHYzMyqyztQfz7wQeAV2PqFXW8tKygzM2tMeZPK6xGxoWNH0mCyz6mYmZltlTepPCjpQmCX9N30vwL+s7ywzMysEeVNKpOAFcCTwP8E7iD7vnozM7Ot8s7+2kL2dcL/Vm44ZmbWyPLO/nqOKmMoEfHOwiMyM7OG1Zu1vzoMBT5ONr3YzMxsq1xjKhGxsuK1PCJ+CBxXcmxmZtZg8nZ/vb9idyeylsvwUiIyM7OGlbf7658rtjcBS4FPFB6NmZk1tLyzv44tOxAzM2t8ebu/vtLd8Yj4QTHhmJlZI+vN7K/DgFlp/2PAHGBZGUGZmVlj6s2XdL0/Il4FkHQJ8KuI+B9lBdboKlc0XnrlR+oYiZlZ7eRdpmUfYEPF/gagpfBozMysoeVtqUwD5km6leyT9f8A3FhaVGZm1pDyzv66XNKdwIdS0bkR8Vh5YZmZWSPK2/0FMAx4JSJ+BLRLGl1STGZm1qByJRVJFwPfAL6ZioYA/1FWUGZm1pjytlT+ATgF+CtARDxPD8u0SNpb0v2SFklaKOnLqbxJ0t2SFqefI1K5JF0jaYmkJyqXhpE0IdVfLGlCRfmhkp5M51wjSb27fTMzK1LepLIhIoK0/L2kN+c4ZxPwjxGxP3AkcL6kA8i+8OveiBgD3Jv2AU4CxqTXROBn6b2agIuBI4DDgYs7ElGqM7HivHE578fMzEqQN6ncJOlfgd0lfQ64hx6+sCsiXoiIR9P2q8AiYBRwKjA1VZsKnJa2TwVujMzD6b32Ak4E7o6IVRGxGrgbGJeO7RoRD6WEd2PFtczMrA7yzv66Kn03/SvAu4FvR8Tded9EUgtwCDAXeFtEvJCu+4Kkt6Zqo9j2E/rtqay78vYq5dXefyJZi4Z99tknb9hmZtZLPSYVSYOA2RHxYbJWQq9IegtwC3BBRLzSzbBHtQOxHeVvLIy4DrgOoLW1tWodMzPbcT12f0XEZmCdpN16e3FJQ8gSys8j4v+m4r+krivSzxdTeTuwd8XpzcDzPZQ3Vyk3M7M6yTumsh54UtLkNMvqGknXdHdCmok1GVjUaRXjWUDHDK4JwG0V5WenWWBHAmtSN9ls4ARJI9IA/QlkLacXgFclHZne6+yKa5mZWR3kXabl9vTqjQ8CnyVLRo+nsguBK8kG/s8D/kz2ffcAdwAnA0uAdcC5ABGxStJ3gUdSve9ExKq0/QXgBmAX4M70MjOzOuk2qUjaJyL+HBFTu6tXTUT8jurjHgBjq9QP4PwurjUFmFKlvA04sLexmZlZOXrq/prZsSHplpJjMTOzBtdTUqlsabyzzEDMzKzx9ZRUoottMzOzN+hpoP59kl4ha7HskrZJ+xERu5YanZmZNZRuk0pEDKpVIGZm1vh6830qZmZm3XJSMTOzwuT98KPtgJZJ//250aVXfqSOkZiZlcstFTMzK4yTipmZFcZJxczMCuOkYmZmhXFSMTOzwjipmJlZYZxUzMysME4qZmZWGCcVMzMrjJOKmZkVxknFzMwK46RiZmaFcVIxM7PCOKmYmVlhvPR9jXkZfDPrz9xSMTOzwjipmJlZYZxUzMysME4qZmZWGCcVMzMrjJOKmZkVxknFzMwK46RiZmaFcVIxM7PCOKmYmVlhvExLHXnJFjPrb9xSMTOzwjipmJlZYUpLKpKmSHpR0oKKsiZJd0tanH6OSOWSdI2kJZKekPT+inMmpPqLJU2oKD9U0pPpnGskqax7MTOzfMpsqdwAjOtUNgm4NyLGAPemfYCTgDHpNRH4GWRJCLgYOAI4HLi4IxGlOhMrzuv8XmZmVmOlJZWImAOs6lR8KjA1bU8FTqsovzEyDwO7S9oLOBG4OyJWRcRq4G5gXDq2a0Q8FBEB3FhxLTMzq5Naj6m8LSJeAEg/35rKRwHLKuq1p7LuyturlFclaaKkNkltK1as2OGbMDOz6vrKQH218ZDYjvKqIuK6iGiNiNaRI0duZ4hmZtaTWieVv6SuK9LPF1N5O7B3Rb1m4PkeypurlJuZWR3VOqnMAjpmcE0AbqsoPzvNAjsSWJO6x2YDJ0gakQboTwBmp2OvSjoyzfo6u+JaDall0u1bX2Zmjaq0T9RLmg4cA+wpqZ1sFteVwE2SzgP+DHw8Vb8DOBlYAqwDzgWIiFWSvgs8kup9JyI6Bv+/QDbDbBfgzvQyM7M6Ki2pRMSnujg0tkrdAM7v4jpTgClVytuAA3ckRjMzK1ZfGag3M7N+wEnFzMwK46RiZmaF8dL3fZCXxDezRuWWipmZFcZJxczMCuOkYmZmhXFSMTOzwjipmJlZYTz7q4/zTDAzayRuqZiZWWGcVMzMrDBOKmZmVhiPqTQQj6+YWV/nloqZmRXGScXMzArjpGJmZoXxmEqD8viKmfVFbqmYmVlhnFTMzKww7v7qB9wVZmZ9hVsqZmZWGCeVfqZl0u3btFzMzGrJScXMzArjMZV+yuMsZlYPbqmYmVlh3FIZANxqMbNacVIZYJxgzKxMTioDmBOMmRXNScUAJxgzK4YH6s3MrDBuqdgbdPXhSbdgzKwnTiqWm7vIzKwnTiq2XZxgzKwaJxXbYe4uM7MOTiolWzr001u3W9b/oo6R1F5PC1s66Zj1P04qBalMHjtSJ4/+kpzyrKbsxGPWWBo+qUgaB/wIGARcHxFX1uq9i0oSfe19+1LS6u0y/k5CZvXV0ElF0iDgJ8DxQDvwiKRZEfFUfSNrbPVKlp1tT3Ir+rtknKTMeqehkwpwOLAkIp4FkDQDOBUoJan0lV+2A0WfeN6X1DuAgl2yppf1d9v+c21AavSkMgpYVrHfDhzRuZKkicDEtLtW0jPb+X57Ai9t57n9iZ9DpvGew6Uq49zGew7l6M/P4R15KzZ6Uqn2tzzeUBBxHXDdDr+Z1BYRrTt6nUbn55Dxc8j4OWT8HDKNvvZXO7B3xX4z8HydYjEzG/AaPak8AoyRNFrSzsB4YFadYzIzG7AauvsrIjZJ+iIwm2xK8ZSIWFjiW+5wF1o/4eeQ8XPI+Dlk/BwARbxhCMLMzGy7NHr3l5mZ9SFOKmZmVhgnlU4kjZP0jKQlkiZVOf4mSb9Mx+dKaql9lLWR41l8RdJTkp6QdK+k3HPZG0lPz6Gi3pmSQlK/nFaa5zlI+kT6O7FQUt9Z76dAOf5d7CPpfkmPpX8bJ9cjzrqJCL/Si2yw/4/AO4GdgT8AB3Sq87+Aa9P2eOCX9Y67js/iWGBY2v5Cf3wWeZ5DqjccmAM8DLTWO+46/X0YAzwGjEj7b6133HV6DtcBX0jbBwBL6x13LV9uqWxr67IvEbEB6Fj2pdKpwNS0fTMwVtIOfEy5z+rxWUTE/RGxLu0+TPY5of4mz98JgO8C3wfW1zK4GsrzHD4H/CQiVgNExIs1jrEW8jyHAHZN27sxwD4756SyrWrLvozqqk5EbALWAHvUJLrayvMsKp0H3FlqRPXR43OQdAiwd0T8upaB1Vievw/7AvtK+r2kh9MK4v1NnudwCfAZSe3AHcD/rk1ofUNDf06lBHmWfcm1NEw/kPs+JX0GaAX+vtSI6qPb5yBpJ+Bq4JxaBVQnef4+DCbrAjuGrNX6W0kHRsTLJcdWS3mew6eAGyLinyV9AJiWnsOW8sOrP7dUtpVn2ZetdSQNJmverqpJdLWVawkcSR8G/g9wSkS8XqPYaqmn5zAcOBB4QNJS4EhgVj8crM/7b+O2iNgYEc8Bz5Almf4kz3M4D7gJICIeAoaSLTY5IDipbCvPsi+zgAlp+0zgvkgjcv1Mj88idfv8K1lC6Y/959DDc4iINRGxZ0S0REQL2djSKRHRVp9wS5Pn38ZMsskbSNqTrDvs2ZpGWb48z+HPwFgASfuTJZUVNY2yjpxUKqQxko5lXxYBN0XEQknfkXRKqjYZ2EPSEuArQJdTTBtZzmfxT8BbgF9JelxSv1t3Ledz6PdyPofZwEpJTwH3A1+LiJX1ibgcOZ/DPwKfk/QHYDpwTj/9j2dVXqbFzMwK45aKmZkVxknFzMwK46RiZmaFcVIxM7PCOKmYmVlhnFTMzKwwTipmDUxSi6QF9Y7DrIOTilkfJGlQvWMw2x5OKmYFkPRdSV+u2L9c0peq1DtG0hxJt6Yvs7o2LUqJpLXpk9lzgQ9IOlTSg5LmS5otaa9U71BJf5D0EHB+re7RLA8nFbNiTCatCZeSxHjg513UPZxsKY+DgHcBp6fyNwMLIuIIYC7wL8CZEXEoMAW4PNX7d+BLEfGBEu7DbId46XuzAkTEUkkr0yKbbwMe62bdq3kR8SyApOnA0WRf+LYZuCXVeTfZ6sd3p++AGwS8IGk3YPeIeDDVmwacVMY9mW0PJxWz4lxP9r0qf0PWsuhK5wX3OvbXR8TmtC1gYefWiKTdq5xv1me4+8usOLcC44DDyFax7crhaen0nYBPAr+rUucZYGT6kickDZH0nvSFV2skHZ3qnVVc+GY7zi0Vs4JExAZJ9wMvV7Q4qnkIuJJsTGUOWTKqdq0zgWtSl9dg4IfAQuBcYIqkdXSfvMxqzkvfmxUktTweBT4eEYu7qHMM8NWI+GgtYzOrFXd/mRVA0gHAEuDerhKK2UDglopZCSQdRDYzq9LrabqwWb/lpGJmZoVx95eZmRXGScXMzArjpGJmZoVxUjEzs8L8f83kv7Fz9jpsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbc7e009f98>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_vs_y(df.y_pred, df.y, kind='hist', bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T03:11:30.694022Z",
     "start_time": "2018-08-04T03:11:22.736469Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEXCAYAAACgUUN5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8nHW59/HPlZnsTZukTWlpWlqgyFZkCSirICJVEVTUAweOgCLiceMc0YP6KOrRx43n6FE8Igcri1rUIhw9LuyLIhZSpLSlQAstNF1ouqRb2iQzcz1/3PekM2mWSTpLMvN9v17zmpl7mfvKPZO55rfcv5+5OyIiUrrKCh2AiIgUlhKBiEiJUyIQESlxSgQiIiVOiUBEpMQpEYiIlDglAikZZjbezL5sZvX7+TrHmNlHsxVX+JoHmdl1Kc/fZWbnZ/MY4eteamanZvt1ZWyLFjoAkdHCzI4FLgB6AAdiwEZgCfB3Dy+6cfdngWczfM1rgIfCfQbk7q8A3xx59P0e+8vAPHd/NeU4P8vmMaQ4KBHImGNmEXeP5+jlt7r798PjVACHAG8DZgO/zMUBc/z3iAxJiUCywsxOBFrc/UcpyxqBjwPfd/eOQfZ9FxAh+BX+OqATeNTdnwnXHwucASwC3gB0AT80sxrgHIIv6yiwCviju+8M9xsHvBOYCewEHh/O3+Tu3cByM+sErjCzQ9z9pWQ8KQnjaOBMYDxBaWKFu99jZv8ITADON7PzgDXufoeZXQ5sAOqBWcCfzawN+IC7fzUlhHIze89g5yQZQ8p5TLj7b1Oqrv7JzBxYGi6/HHjZ3R8L9zkAmAtMBXYDfwf+4u6JsArtGuBu4LTwb1kD3OPuO4ZzLmV0UyKQbHkWOMfMprn72nDZcQRfOgMmgRRHAfeEt1nAxWa22d3XhOvrgTrgBwBmZsBFwCbgv4A48HbgQuC2cJ8LCZLGdwk+6+8fyR/m7q+Y2Y4wrpdS15lZOfAe4A53XxWWIqaG+/1ikKqh44A7CUoZUWBaP4ce6pwMFvOPwqqhO1KrhvrEXgV8AHgS+DnQAPwjwblMTZpHAT8Nl18KnAX8dqgYZOxQY7Fkhbt3AUsJvuAwszLgWODpDF+izd2fdfeEu78EPBfun5QAHnD3HnfvIfiynQr8wd33hMvuB2aFjcLjCb487wvX7wQe3Y8/cTtQM8C6ODDJzKrdvTus7x/Kc+6+ygM9A2wz1DnZX7MJYn/M3WPu3k6QAI7vs92j7t4ZvsdLgAOzGIOMAioRSDYtAj5gZvcCBxP80Hghw337lho6CH9Zh3a4eyzleQPB5/faoHDQK0ZQhZEcTTH1dbdmGEt/xhNUPaVx9x4z+zlwMnC2mW0F/uruS4Z4vUxKSUOdk/01AehINoKHthD8ralSq4G6gYosxiCjgBKBZI27rzWzLQRVCYcDzwyjEbRvl856gl/hvS/fZ30HwZfSt/p8kQFBV9GU19kSPm7IMJa+rzWDoFpqn0QA4O6rgdVhKeh1wPvNbK27b+kn7t7dMjj0YOekvy/kOmDbMI6xDZhgZpZyDhtIP+9SAlQ1JNm2iODX8WwyrxYCaDazOWZWZmazgCOBxYNsv46gwfVtYaMxZlYbNtzi7tuB1QTtFpVhw/EZw/lDzKzCzA4naGt4Pqye6bvNODM70syq3D0B7AlXJcL7ncDE4Rw3xWDnZANQa2aHWeAI4KA++w917BUEPwZPN7OImU0iaBQezvsmRUAlAsm2JcBbCXrIbB7GfssIksd5BL1Xfj9QIyeAu7uZ3Qm8GbgqTAa7CBpzl4ab3UXQa+hf2dtrqO+XZV8NZvb58HHyOoI/M/CXowEnEvQMKiP4lX1PSgP5Y8DbzewNBHX+w+nHP+A5cfctZvbH8O8rD7dd3mf/h4CzzOxcYJm7/y51pbvvMbM7CHoNnUKQxJ4BnhhGjFIETBPTSDaFvXk+BTyYQT15cp/ebo85DU5E+qWqIcm2OQTXBDxX6EBEJDOqGpKsMbPPEtSN/09qI7GZHUNQvdGf/81HbCIyMFUNiYiUOFUNiYiUuDFRNTRp0iSfOXNmocMQERlTFi1atMndm4babkwkgpkzZ9La2lroMERExhQzy2S4E1UNiYiUOiUCEZESp0QgIlLixkQbgYiUlp6eHtra2tizZ8/QGwtVVVU0NzdTXl4+ov2VCERk1Glra6Ouro6ZM2fSZ5hx6cPd2bx5M21tbcyaNWtEr6GqIREZdfbs2cPEiROVBDJgZkycOHG/Sk85SwRmNs/MNprZ0j7LP2FmL5jZMjP7dq6OLyJjm5JA5vb3XOWyRHArwfC2vczsLOAC4Bh3Pwq4IYfHlxFKJJz7n3uNzu7Y0BuLyJiXs0Tg7o+xd2aopI8C3wznPsXdN+bq+DJydz61hg/f3spdT68demORInb33XdjZjz//PODbnfrrbeybt26ER/nkUce4bzz+h+X8bbbbmP27NnMnj2b2267bcTHGEy+2wgOI5gNaaGZPWpmJw60oZldZWatZtba3t6exxBl/bbdALy2TT02pLTNnz+f0047jTvvvHPQ7fY3EQxky5YtfOUrX2HhwoU8+eSTfOUrX2Hr1v2Zert/+U4EUYI5Ud8IfAb4lQ1QueXuN7t7i7u3NDUNOVSGZNGurmAE6c27ugociUjh7Ny5k8cff5yf/OQnaYng29/+NnPmzOH1r3891113HQsWLKC1tZVLLrmEY489lt27dzNz5kw2bdoEQGtrK2eeeSYATz75JKeccgrHHXccp5xyCi+88MKgMdx7772cc845NDY20tDQwDnnnMOf/vSnrP+t+e4+2gb8Jpwo+0kzSwCTAP3kH0U6dncDsGVXd4EjEYGv/G4Zz63bntXXPPLA8Vz/zqMG3eaee+5h7ty5HHbYYTQ2NvL000/z2muvcc8997Bw4UJqamrYsmULjY2N3Hjjjdxwww20tLQM+pqHH344jz32GNFolAceeIDPf/7z3HXXXWnbtLa2ctNNN3HLLbewdu1apk+f3ruuubmZtWuzX2Wb70RwD8Ecs4+Y2WFABbApzzHIELZ19gB7SwYipWj+/Plcc801AFx00UXMnz+fRCLBFVdcQU1NDQCNjY3Des1t27Zx2WWXsWLFCsyMnp6efbZpaWnhlltuAYJrBPrKRW+qnCUCM5sPnAlMMrM24HpgHjAv7FLaDVzmmhln1NndEySAnV3qNSSFN9Qv91zYvHkzDz30EEuXLsXMiMfjmBkXXnhhRl/E0WiURCIBkNa//4tf/CJnnXUWd999N6tXr+6tMhpIc3MzjzzySO/ztra2IfcZiVz2GrrY3ae6e7m7N7v7T9y9290vdfej3f14d38oV8eXkeuOBR/gXUoEUqIWLFjABz7wAV555RVWr17NmjVrmDVrFo2NjcybN4/Ozk4gaMwFqKurY8eOHb37z5w5k0WLFgGkVf1s27aNadOmAUED81DOPfdc7rvvPrZu3crWrVu57777OPfcc7P1Z/bSlcWyj+54kAg6u1U1JKVp/vz5vPvd705bduGFF7Ju3TrOP/98WlpaOPbYY7nhhuBSqMsvv5yrr766t7H4+uuv51Of+hSnn346kUik9zU++9nP8rnPfY5TTz2VeLz//6/W1lauvPJKIKh6+uIXv8iJJ57IiSeeyJe+9KVhV0dlYkzMWdzS0uKamCZ/5n7vMZ7fsIMJ1eUsvv6thQ5HStDy5cs54ogjCh3GmNLfOTOzRe4+eAs2KhFIP1KrhsbCDwUR2T9KBLKPrjARxBLe+1hEipcSgewj2UYAajAWKQVKBLKP7liCuqqgZ7GuJRApfkoEso/uWILG2gpA1xKIlAIlAtlHdzxBfU2QCHZpKGqRoqdEIGniCSeecOqrg7lPd+taAilho2EY6rlz51JfXz/g+mxQIpA0ya6jE5KJoEeJQEpXoYehBvjMZz7DHXfckZPXTlIikDR9E8EeJQIpUaNhGGqAs88+m7q6upz8jUn5Hn1URrmu8LL3+hpVDcko8cfrYMOS7L7mlDnwtm8OusloGIY6X5QIJE1XT3qJQOMNSakaDcNQ54sSgaRJXkw2Xm0EMloM8cs9F0bLMNT5ojYCSZNsIxhXGaXM1EYgpWm0DEOdLzlLBGY2z8w2hpPQ9F13rZm5mU3K1fFlZJKJoCJSRnV5RG0EUpJGyzDUAKeffjrve9/7ePDBB2lububee+/N+t+bs2GozewMYCdwu7sfnbJ8OnALcDhwgrsPOVWlhqHOn6dWb+F9Nz3B7R88iX/91TOce9QUvv7uOYUOS0qMhqEevlE5DLW7PwZs6WfVd4HPAhrfeBTqLRFEy6hSiUCkJOS1jcDMzgfWuvviDLa9ysxazay1vb09D9EJpCeCmoqIGotFSkDeEoGZ1QBfAL6UyfbufrO7t7h7S1NTU26Dk1498T5tBEoEUiCaFClz+3uu8lkiOASYBSw2s9VAM/C0mU3JYwwyhHgi+EBFykxVQ1IwVVVVbN68WckgA+7O5s2bqaqqGvFr5O06AndfAkxOPg+TQUsmjcWSP/HwHy9aZlRXRNi6q7vAEUkpam5upq2tDVULZ6aqqorm5uYR75+zRGBm84EzgUlm1gZc7+4/ydXxJDtSSwTV5RHWqWpICqC8vJxZs2YVOoySkbNE4O4XD7F+Zq6OLSMXi6ckgoqIhpgQKQG6sljSJKuGkiUCXVksUvyUCCRN36ohNRaLFD8lAkkTS6RXDe3uiavnhkiRUyKQNIlEstdQcGVxwveOSCoixUmJQNL0lggsqBoCTU4jUuyUCCRNskQQiRg1FWEiUIOxSFFTIpA0sUT6BWWgEoFIsVMikDTxcFalMguGmACVCESKnRKBpEm2C0fL9rYR6FoCkeKmRCBp4okEZlCWUjWkq4tFipsSgaSJJZxIODm3eg2JlAYlAkkTdydSFiYC9RoSKQlKBJImHneiZeklArURiBQ3JQJJE0s4ZWWqGhIpJUoEkibhKSWCZGOxSgQiRU2JQNLEEnvbCCqjZZjBHpUIRIpazhKBmc0zs41mtjRl2XfM7Hkze9bM7jaz+lwdX0YmHt+bCCwcb0iNxSLFLZclgluBuX2W3Q8c7e7HAC8Cn8vh8WUE4u5Ey/Z+LJQIRIpfzhKBuz8GbOmz7D53j4VP/waMfLZlyYl4wknJA1SVR9jdrWGoRYpZIdsIPgj8caCVZnaVmbWaWWt7e3sewypt8USfEkFFhN09sUH2EJGxriCJwMy+AMSAnw+0jbvf7O4t7t7S1NSUv+BKXDylsRjQdJUiJSCa7wOa2WXAecDZrjkQR51YItE7xASojUCkFOQ1EZjZXODfgDe5e2c+jy2ZiSdILxFUROjY3VPAiEQk13LZfXQ+8ATwOjNrM7MPATcCdcD9ZvaMmd2Uq+PLyMQTCaKR9BKBriMQKW45KxG4+8X9LP5Jro4n2RFLOGWWXiLoVGOxSFHTlcWSJnWICVD3UZFSoEQgaWLxfXsNafRRkeKmRCBp+nYfrakIeg2pg5dI8VIikDSpE9NA0EYQTzg9cSUCkWKlRCBp+pYIqso1S5lIsVMikDSxeHpjsSanESl+SgSSJrFP1VDwEVGJQKR4KRFImlg/Yw2BSgQixUyJQNIkEk4kbfTR4JpDlQhEipcSgaSJJfpvI9C1BCLFS4lA0sT7DjERJoJOVQ2JFC0lAkkT71siUGOxSNFTIpA0sYQTiex7HYFGIBUpXkoEkibeZ2KaGjUWixQ9JQJJ099UlaBEIFLMlAgkTd82gspo8BFRY7FI8crlDGXzzGyjmS1NWdZoZveb2YrwviFXx5eR6XtBWVmZUVVepu6jIkUslyWCW4G5fZZdBzzo7rOBB8PnMor0HWICwgnsVSIQKVo5SwTu/hiwpc/iC4Dbwse3Ae/K1fFlZPqWCCBMBCoRiBStfLcRHODu6wHC+8kDbWhmV5lZq5m1tre35y3AUpZIOO7smwgqlAhEitmobSx295vdvcXdW5qamgodTkmIh7OQRftLBKoaEila+U4Er5nZVIDwfmOejy+DiCeCRFCmNgKRkpLvRPBb4LLw8WXA/+T5+DKIZCLoWyKoUhuBSFHLZffR+cATwOvMrM3MPgR8EzjHzFYA54TPZZSIhYkgdRhqCEoE6j4qUryiuXphd794gFVn5+qYsn+SJYJIeoGA6oqILigTKWKjtrFY8q83EUTSPxa1lVE6u2OFCElE8kCJQHoN1EZQVxll+x4lApFilVEiMLO7zOwdZqbEUcRiiQRA2uijAHVVUbpjCbpiqh4SKUaZfrH/CPhHYIWZfdPMDs9hTFIgYR7Y54KycZVBU9KuLiUCkWKUUSJw9wfc/RLgeGA1cL+Z/dXMrjCz8lwGKPmTLBFE+7QWj6sK3uKdqh4SKUoZV/WY2UTgcuBK4O/AfxIkhvtzEpnkXe8FZdZ/iWD7np68xyQiuZdR91Ez+w1wOHAH8M7keEHAL82sNVfBSX4NNMREXVXwMdnZpRKBSDHK9DqCW9z9D6kLzKzS3bvcvSUHcUkBxOL9DzHRmwhUNSRSlDKtGvpaP8ueyGYgUngDdR9NVg2pRCBSnAYtEZjZFGAaUG1mxwHJb4jxQE2OY5M8S1YN7dNrKCwR7FAbgUhRGqpq6FyCBuJm4D9Slu8APp+jmKRAeq8s3ueCsqDX0A6VCESK0qCJwN1vA24zswvd/a48xSQFMlAiqCovI1pmaiMQKVJDVQ1d6u4/A2aa2b/2Xe/u/9HPbjJG7W0jSG86MjPGVUXVRiBSpIaqGqoN78flOhApvL3DUO+7blxllB0qEYgUpaGqhn4c3n8lP+FIISUGmI8AlAhEilmmg85928zGm1m5mT1oZpvM7NKRHtTM/sXMlpnZUjObb2ZVI30tyZ7YAN1HAcZXlbOzS72GRIpRptcRvNXdtwPnAW3AYcBnRnJAM5sGfBJocfejgQhw0UheS7IrHo411HeICUBtBCJFLNNEkBxY7u3AfHffsp/HjRJcmxAluB5h3X6+nmRBPBx9tO+gcxBUDanXkEhxyjQR/M7MngdagAfNrAnYM5IDuvta4AbgVWA9sM3d7+u7nZldZWatZtba3t4+kkPJMMWGKBGojUCkOGU6DPV1wMkE1Tk9wC7ggpEc0Mwawn1nAQcCtf21N7j7ze7e4u4tTU1NIzmUDNNAQ0xAMEuZLigTKU7Dmbz+CILrCVL3uX0Ex3wLsMrd26F3ZNNTgJ+N4LUkiwa6oAzSZymrjEbyHZqI5FCmw1DfARwCPAMkp6lyRpYIXgXeaGY1wG7gbEBDWY8CgyWC8dVBM9H23TGa6pQIRIpJpiWCFuBI93BUsv3g7gvNbAHwNBAjmOTm5v19Xdl/g3Ufra+pAKCjs5umusq8xiUiuZVpIlgKTCFo3N1v7n49cH02XkuyJzHA6KMA9WGJoGO3riUQKTaZJoJJwHNm9iTQlVzo7ufnJCopiOTENP0mgpowEXQqEYgUm0wTwZdzGYSMDoOVCBrCqqGtnd15jUlEci+jRODuj5rZQcBsd38gbOhVi2GRiQ0w+ijAhLBEsE0lApGik+lYQx8GFgA/DhdNA+7JVVBSGMleQ/3kAeoqo0TKjI7dKhGIFJtMryz+GHAqsB3A3VcAk3MVlBTGQPMRQDAnQX11OVtVIhApOpkmgi537/0pGF5Utt9dSWV0SVYN9dNEAATVQ6oaEik+mSaCR83s8wQDxZ0D/Br4Xe7CkkKIJxJEygzrZ6whCLqQqmpIpPhkmgiuA9qBJcBHgD8A/ydXQUlhxBP99xhKaqipYOsulQhEik2mvYYSZnYPcE9yjCApPvFEgsgApQEIqoae37AjjxGJSD4MWiKwwJfNbBPwPPCCmbWb2ZfyE57kUyzh/Q4vkdRQU0GHriMQKTpDVQ1dQ9Bb6ER3n+jujcAbgFPN7F9yHp3kVSLhRPqZlCapvrqcXd1xumOJPEYlIrk2VCL4AHCxu69KLnD3l4FLw3VSRGIJH7RqqHeYCTUYixSVoRJBubtv6rswbCco72d7GcPiCR+0sTg5Aqm6kIoUl6ESwWA//fSzsMjEh2gjaKwNEsHmXXrrRYrJUL2GXm9m2/tZbkBVDuKRAoonnLJBEkFyHoJNO7sG3EZExp5BE4G7a2C5EhL3wUsEk8YFiaB9hxKBSDHJ9IKyrDKzejNbYGbPm9lyMzu5EHFIuthQbQTV5UTLTIlApMgMZ/L6bPpP4E/u/l4zqwBqChSHpIjHB08EZWXGpHGVSgQiRSbvicDMxgNnAJcDhIPZqfVxFIi7E+lvDOoUTXWVtKuNQKSoFKJq6GCCcYt+amZ/N7NbzKy270ZmdpWZtZpZa3u7RrXIh6F6DUGYCFQiECkqhUgEUeB44Efufhywi2BQuzTufrO7t7h7S1NTU75jLEmxIXoNATSpakik6BQiEbQBbe6+MHy+gCAxSIElMiwRbN7VTSKh6ShEikXeE4G7bwDWmNnrwkVnA8/lOw7ZV2yI0UchSATxhGsSe5EiUqheQ58Afh72GHoZuKJAcUiKoYaYgL0XlbXv7GJieF2BiIxtBUkE7v4M0FKIY8vAYgmnqnzwawh7E8GOLg6fko+oRCTXCnJBmYxOGfUaCksBG7erwVikWCgRSK+e+NDXEUyZEAwxtX7b7nyEJCJ5oEQgveKJxJAlgqryCJPGVbC2Q4lApFgoEUivWMKJDjJDWdK0+mratioRiBQLJQLpFYsP3UYAMK2hWiUCkSKiRCC94gknGhn6IzGtvpp1Hbtx10VlIsVAiUB69cSHbiMAOLC+mj09Cc1UJlIklAikVyYXlEFQIgBYp+ohkaKgRCC9YgmnPJOqoYYgEaxVg7FIUVAikF6xeGJYJQI1GIsUByUC6ZVp99EJ1eWMq4yyZktnHqISkVxTIpBesQyGmAAwM2ZOqmHVZiUCkWKgRCAAuHvYWJzZR2LWpHGs3rQrx1GJSD4oEQgQ9BgCKM+gRAAwa1ItbVs76YrFcxmWiOSBEoEAQbUQQCSDNgKAWZNqSDhqJxApAgVLBGYWCSev/99CxSB7xXpLBJlXDQGs2qREIDLWFbJE8ClgeQGPLyni8bBEkGnV0MRaAFZt2pmzmEQkPwqSCMysGXgHcEshji/76kkkADLqPgowoaacibUVrFKDsciYV6g5i78HfBaoK9DxpY9kY3F0sKqh3Vth4c2wfS0c+hYOmdTIyo0qEYiMdXlPBGZ2HrDR3ReZ2ZmDbHcVcBXAjBkz8hRd6eqJhyWCgaqGdrbDvLfCllVQNR6evo1vjDuBy3b+M+6OWWYlCREZfQpRNXQqcL6ZrQbuBN5sZj/ru5G73+zuLe7e0tTUlO8YS05viWCgqqE/fha2rYUr/gifXQXnfY+ZnUv4ceKrbNiwLo+Riki25T0RuPvn3L3Z3WcCFwEPuful+Y5D0vV2H+2vRLBxOSz7DZx2DRx0MpRFoOUKXn7LzRxq66i+658gpiGpRcYqXUcgQDA7GQzQRvDkf0OkEk76SNriKcefx6d7rqZ+0yL4w7X5CFNEcqCgicDdH3H38woZgwRiA/UainXBs7+Co98DtRPTVtVVlbO4/s3c13gJPH0bLL4zX+GKSBapRCBAaomgTyJY/Rfo3gFHvbvf/Q6fMp7vdF8IM06B338aNr+U61BFJMuUCATY20awz5zFL/4JotUw64x+95szbQIrN+9hxzv+K2g7uOtKtReIjDFKBAKkXkfQp0Sw4n44+Ewor+53v+NnNOAOz2wfB+f/ANY9DQ9/PbfBikhWKREIEMxOBn16De14DbaugpmnDbjf66dPwAyefqUDjrwATrgcHv8erHwwxxGLSLYoEQiQMuhcamPxmoXB/fQ3DLhfXVU5h02u4+lXtwYLzv0GNB0Bd38kSCQiMuopEQiwt9dQ2sQ0axYG3Uanvn7QfY8/qJ5n1nSQSDhU1MD7boWunfCbD0NC8xWIjHZKBAIM0Gvo1b/BtOMhWjHovsdNb2Db7h5eTg5AN/lwePu3YdWj8Of/yFXIIpIlSgQC9DPERM8eWL8Ypp805L7HH9QAwFOrt+xdeNw/wZz3BQ3HyzXlhMhopkQgAPT07TW08TlI9MC0E4bc95CmWg4YX8njKzftXWgG7/x+UKK460pYuygXYYtIFigRCAA9saCNoDx5HcFrS4P7A44ecl8z49RDJ/HXlzYH7QRJFTVw8Z0wrgl+8Q/Q/kK2wxaRLFAiEAC6w+6jFdHwI7FhCVSMg4ZZGe1/2qGT2LKrm+fWb09fMW4yXHIXWBnc+o5gADsRGVWUCASA7rBEUBmNBAs2LIEDjoIM5zA+7dBJADy2on3flU2HweW/B4vAvLmw6s9ZiVlEskOJQADoigXdPCuiZZBIwIalMGVOxvtPHl/FnGkTuHfZANcOTJoNH/wTjDsA7ngXPH17NsIWkSxQIhBgb4mgIlIGHa8EA80NIxEAvH3OVBav6aBta2f/GzTOgivvD8Yt+u0n4PfXBqObikhBKREIsDcRlEcspaF4uIlgCgB/Wrph4I2qJsA//hpO/jg89d/w07dBx5oRxSwi2aFEIAB0xRNURsuCuYc3LAkadycfMazXOGhiLUcdOJ7fL1k/+IaRKJz7dXj/7dD+Ivz4DI1NJFJAeU8EZjbdzB42s+VmtszMPpXvGGRfXT2JlB5DS2HioUH3z2F6xzFT+furHaxKXmU8mCMvgKsegbop8LML4ZFvBe0TIpJXhSgRxIBPu/sRwBuBj5nZkQWIQ1J0hyUCIOwxNPT1A/157/HNRMqMO596NbMdJh0KVz4Ax/wDPPJ/4Rfvg84tQ+8nIllTiMnr17v70+HjHcByYFq+45B03bFE0FC8uwO2vTrshuKkyeOrOPvwySxobettdxhSRS28+yY477uw6jH4rzfC8t+N6PgiMnwFbSMws5nAccDCftZdZWatZtba3t5P33TJqu5YgsryCLy2LFgwwhIBwCVvPIjNu7q555m1me9kBi0fhCsfDC5C++Wl8KsPwPYh2htEZL8VLBGY2TjgLuAad9/ed7273+zuLe7e0tTUlP8AS0xviSCZCKaMPBGcMXsSR04dz02PvpQ+5EQmph4DH34Y3vxFeOFPcOOJ8MQPIR4bcTwiMriCJAIBAnGAAAASoUlEQVQzKydIAj93998UIgZJ1xWLB43Fry2B6kaomzri1zIzPnrmIbzcvov7nhukK+lAIuVwxrXwsb/BjDfAvZ+Hm98UDIstIllXiF5DBvwEWO7uGqx+lOiOh72GNiwNSgNmQ+80iLfPmcqsSbX8v/te7J0Gc9gaD4ZLFsD774DdW2HeufA/H4Ndm/crNhFJV4gSwanAPwFvNrNnwtvbCxCHpOiOJaiKeDAo3DAvJOtPpMz4t7mHs2LjTuY/mWEPov6YwZHnw8eehFM/BYvvhBtPgKUqSIpkSyF6Df3F3c3dj3H3Y8PbH/Idh6TrjiWYwXqI7d6v9oFU5x51ACcfPJFv3/sCazt279+LVY6Dc74KV/8FGg+BBVfA3VfDnn2al0RkmHRlsQDQFUswM7Y6eLIfPYZSmRnfuvAYEgnnX375zMiriFJNPiIYvO5N/wbP/hJuOg1e3afTmYgMgxKBAEGJYGbsZSiLQtPrsva6MybW8O/vOponV23ha7/P0lwEkXI46/NwxR8Bh5/OhYf/r3oWiYyQEoEAQYlgevdLMOkwiFZm9bXfc3wzHzptFrf+dfX+tRf0NeONcPXjwVXJj34raEzWLGgiw6ZEIAB0x+LM2P18MMdwDnzubYdzxmFNfPGepSx8OYu9fqrGB1clv3cebF4RXJV814c1E5rIMCgRCABNsXWMi3fAtJacvH40UsYPLj6OGRNr+PDtrTy1OsvjCR19IXx8UTC89fP/GySEeXPhmfnQtTO7xxIpMkoEgrtzeCysUmk+MWfHmVBdzu0fPIlJ4yq59JaFPPDcALOZjdS4Jnjrv8M1S+Gcf4edG+Geq+E7h8AvLoK//wx2jOACN5EiFy10AFJ43fEEx7CCnrJqyoc5B8FwNTfU8OurT+aDtz7FR362iK+962guPmlGdg9SOxFO/SSc8gl45a+w/Lew/H/hxT8G6yfOhpmn7b3VTcnu8UXGGHMf5lgwBdDS0uKtra2FDqNobdnVzZpvvYHJEycy9VMP5OWYu7pi/PPPn+bRF9v5+FmH8um3HhZMipMr7rB+cTC66eq/wKtPQFd4DcLE2TDrdJh5epAYxk3OXRwieWRmi9x9yPpelQiEzu1bOdJeYeXEsxn5CEPDU1sZ5ZbLWvjiPUu58eGVrNq8i69dcDQNtRW5OaAZHHhscDv1k5CIw4Zng6Sw6s/w7K+hdV6wbdPhQVKYdUZwq67PTUwio4QSgeCr/0y5xdl+4Gl5PW55pIxvvGcOMybW8B/3vcjjKzfx0TcdwgdOnkl1RSS3By+LwIHHBbdTPhFcg7B+Max+LEgMz/wimFPZIkE31dnnwOy3wuQj93scJpHRRlVDwsY7P07t8l/x9EVPc/oRzQWJ4fkN2/nGH57n0Rfbaaqr5ONnHcpFJ02nMprjhDCQeA+0tcLKB2DFfUHpAWD8tCApHHoOTH9D0EAtMkplWjWkRFDq3Om84Wie2N7E5I/8D3OaJxQ0nKdWb+GGe19g4aotHDihik+ePZsLT2imPFLgDm7b1+9NCi89DN07guV1U4PZ3KYcE97PgYZZUKYOeVJ4SgSSmTVPwU/ewrU9H+Gaa79Mc8PwJ6zPNnfn8ZWb+c59L7B4TQczJ9Zw8UkzOPuIyRzSNC63jcqZiHVD21Ow7u/B/M4bng2uaPZ4sL5iHBxwVJAUDjg6uJ98JFQU/txKaVEikMz8/lpii27n2M4f8uRX301NxehpNnJ3Hly+kR88vJLFazoAmDSukpMPmchbjpjMaYdOYuK47A6HMWI9u2Hjc8F8Dq8tDRPE0r0lBysLRk2dMicY3fWA8L5uqtocJGfUa0iG1rkFFs9nef2ZxHvqqC4vUH38AMyMtxx5AG858gDWbOnk8ZWbeOLlzTy+cjO/W7wOgGn11cyZNoE5zROC+2kTctfzaDDl1TDthOCW5A4dr+xNChuWwNpWWJYyl0K0GiY0Q/304H5Cyn399OC+bHS9L1J8lAhK2ePfg+5dLKh5P9Mbqwtf5TKI6Y01XHTSDC46aQaJhPNMWweLVm/l2bXbWLp2G39atveK4QMnVHHE1PEcNqWOQ5vGMb2xhoaacmoqo9RVRRlfVZ6foM2gYWZwO+Kde5fv7gjmhn5tWZAotq2BjjVBsti1Mf01ysqD/SceEpQoJh4c3h8C45vVFiFZUZBEYGZzgf8EIsAt7v7NQsRR0l5+FP56I7z+Yh5f1cTMiWOn/rqszDh+RgPHz2joXbZtdw/L1m5jydptLFu3nRc27ODRF9uJJfat+qyrijKjsYbpDTVMb6xmemMNk8ZVUl9dTn1NBQ215TTUVFCVqxJSdT3MPDW49dWzB7avDZLD1ldgy8uw5SXY/HLwnsVSJviJVELjrL0Jom4q1E4OejLVNgWPq+uDYbtFBpH3RGBmEeCHwDlAG/CUmf3W3Z/LdywlafdW+PvP4aGvwaTD2HTaV1i58CkueP2BhY5sv0yoLueUQydxyqGTepf1xBO8uqWTtVt3s31PD7u6Ymzb3UPb1t28uqWTFRt38PALG+mK9T9hTmW0jIaaCuprgsRQWxmhqjxCdXmEmooIVRXB4+ryCJXRMiqT99Hk872PK6JllEeMSFkZ0TIjGjGifR5HyozyaCU2MfzF31ciATvWh4nhpb0JYstLQY+meFf/JydaHYzSWjl+731lXfh4AlTUBkOPR6uC+/LqvY9776v7PK+C8qrgPlKhdo4xrhAlgpOAle7+MoCZ3QlcAGQ/EfztR/D87wF4qX0ne3qCXh1G8CvRgfSPrxOu6t2mdznJ5cPZtv+GeEvZN+Ptk/H6YNt7nxjTX7/WdzHZgyGgn4wcxzc7r+HVHy+mzOBtc/J1TXH+lEfKOKRpHIc0jRtwm0TC2bSziy2d3Wzd1UNHZzcdu3vY2tlNR2fwfGt4v35bD7t74uzujvfeD5RE9keZBaO1RsuMSJmF92VEyiBiRlmZES2bSZnNAgvf72qnzjppSHTQ6B3U00FDooM630Wt76K2u5Parl3UbNtNra+j1jup8V3UeifV7NnvmLupoNuCW0/yRjndVoFb/9VXzkDJo//l/W0/cFeXAV5jFCWshppyptVXD73h2V+G5hOG3Gx/FCIRTAPWpDxvA97QdyMzuwq4CmDGjBEOSpaIBzcAj+/9ErW9HyBP+2ov6/3HSiSX9/ncpH8Yg8fpH67+H6d9YHu3T11v/azfu9xIT1ze+w2Qvu2+6/du85pV8reK6SyvbeHVytdxoBnTzHjHnCkcOnngL8tiVlZmTB5fxeTxVSPaP55w9vQECaErFqerJ0FXLEF38nnK8ljCiSUSxOIePI4Hy+KJ9OfJ9fFEgp54sE88ESStuDuJcPuEe5/fExNwptDp0Ams9eQqxz1lMwifhz8wEk6EHsq9m2iiO7j3bsoTXeHjYF3y+b63LsqT24S3ipRllvLfljTw1/EAP4bc91lng6SSfl8j7QdY4ZUlgHgmX8G5D7oQiaC/926fv9TdbwZuhqD76IiOdMrHgxvQT0G7ZJ1f6ACKSKTMqK2MUjtKerGKjEQhuhy0AdNTnjcD6woQh4iIUJhE8BQw28xmmVkFcBHw2wLEISIiFKBqyN1jZvZx4F6C7qPz3H1ZvuMQEZFAQa4jcPc/AH8oxLFFRCSdLksUESlxSgQiIiVOiUBEpMQpEYiIlLgxMR+BmbUDrwyyySRgU57CGamxECOMjTjHQowwNuIcCzHC2IhzNMZ4kLsPOZ/qmEgEQzGz1kwmXyiksRAjjI04x0KMMDbiHAsxwtiIcyzEOBBVDYmIlDglAhGRElcsieDmQgeQgbEQI4yNOMdCjDA24hwLMcLYiHMsxNivomgjEBGRkSuWEoGIiIyQEoGISIkbM4nAzBrN7H4zWxHeNwywXdzMnglvv01ZPsvMFob7/zIcAjvvMZrZsWb2hJktM7NnzewfUtbdamarUuI/NouxzTWzF8xspZld18/6yvC8rAzP08yUdZ8Ll79gZudmK6YRxvmvZvZceO4eNLODUtb1+94XIMbLzaw9JZYrU9ZdFn4+VpjZZbmKMcM4v5sS44tm1pGyLl/ncp6ZbTSzpQOsNzP7fvg3PGtmx6esy8u5zCDGS8LYnjWzv5rZ61PWrTazJeF5bM1VjPvN3cfEDfg2cF34+DrgWwNst3OA5b8CLgof3wR8tBAxAocBs8PHBwLrgfrw+a3Ae3MQVwR4CTgYqAAWA0f22eafgZvCxxcBvwwfHxluXwnMCl8nkqP3OJM4zwJqwscfTcY52HtfgBgvB27sZ99G4OXwviF83FCoOPts/wmCIeHzdi7D45wBHA8sHWD924E/Esxs+EZgYQHO5VAxnpI8NvC2ZIzh89XApHycy/25jZkSAcEE97eFj28D3pXpjmZmwJuBBSPZfxiGjNHdX3T3FeHjdcBGYMgr//bTScBKd3/Z3buBO8NYU6XGvgA4OzxvFwB3unuXu68CVoavV5A43f1hd+8Mn/6NYIa7fMrkXA7kXOB+d9/i7luB+4G5oyTOi4H5OYplQO7+GLBlkE0uAG73wN+AejObSh7P5VAxuvtfwxigMJ/J/TaWEsEB7r4eILyfPMB2VWbWamZ/M7PkF/FEoMPdY+HzNmBaAWMEwMxOIvi19lLK4q+HRczvmlm2ZsKdBqxJed7f39+7TXiethGct0z2zZbhHutDBL8Wk/p777Mt0xgvDN/HBWaWnJp1VJ7LsHptFvBQyuJ8nMtMDPR35PNcDkffz6QD95nZIjO7qkAxDakgE9MMxMweAKb0s+oLw3iZGe6+zswOBh4ysyXA9n62G1G/2SzFSPir5g7gMndPhIs/B2wgSA43A/8GfHUkcfY9XD/L+v79A22Tyb7ZkvGxzOxSoAV4U8rifd57d3+pv/1zHOPvgPnu3mVmVxOUtN6c4b7ZMpxjXQQscPd4yrJ8nMtMjIbPZUbM7CyCRHBayuJTw/M4GbjfzJ4PSxijyqhKBO7+loHWmdlrZjbV3deHX6IbB3iNdeH9y2b2CHAccBdBkTIa/tptBtYVKkYzGw/8Hvg/YXE3+drrw4ddZvZT4NqRxNiPNmB6yvP+/v7kNm1mFgUmEBSHM9k3WzI6lpm9hSDxvsndu5LLB3jvs/3lNWSM7r455el/A99K2ffMPvs+kuX4kobzvl0EfCx1QZ7OZSYG+jvyeS6HZGbHALcAb0t9/1PO40Yzu5ugym7UJYKCN1JkegO+Q3pD7Lf72aYBqAwfTwJWEDaQAb8mvbH4nwsUYwXwIHBNP+umhvcGfA/4ZpbiihI0ps1ib8PhUX22+RjpjcW/Ch8fRXpj8cvkrrE4kziTX0izM33vCxDj1JTH7wb+Fj5uBFaFsTaEjxsLdS7D7V5H0KBp+T6XKcebycANse8gvbH4yXyfywxinEHQdnZKn+W1QF3K478Cc3MV4379fYUOYBhvxMTwC3RFeN8YLm8BbgkfnwIsCT/0S4APpex/MPBk+Ib9OvlBL0CMlwI9wDMpt2PDdQ+FcS8FfgaMy2JsbwdeDL9EvxAu+ypwfvi4KjwvK8PzdHDKvl8I93uB4BdPLt/noeJ8AHgt5dz9dqj3vgAxfgNYFsbyMHB4yr4fDM/xSuCKQp7L8PmX6fODI8/ncj5Bz7kegl/5HwKuBq4O1xvww/BvWAK05PtcZhDjLcDWlM9ka7j84PAcLg4/D1/I5fu9PzcNMSEiUuLGUq8hERHJASUCEZESp0QgIlLilAhEREqcEoGISIlTIhARKXFKBCIFYGYzBxrWWCTflAhEssjMIoWOQWS4lAikpJnZv5vZp1Kef93MPtnPdmea2WNmdnc4Mc5NZlYWrttpZl81s4XAyWZ2gpk9Go44eW847hTh8sVm9gR9xvYRKSQlAil1PwEuAwi/2C8Cfj7AticBnwbmAIcA7wmX1xKMQ/MGYCHwA4IJhk4A5gFfD7f7KfBJdz85B3+HyIiNqtFHRfLN3Veb2WYzOw44APi7p48emupJd38ZwMzmEww3vACIE4xwC8EgbkcTDDkMwUxh681sAsFMdI+G291BMJuVSMEpEYgEg4ZdTjDPxLxBtus7MFfy+R7fO5a/Acv6/uo3s/p+9hcZFVQ1JAJ3E0xzeCJw7yDbnWRms8IqpH8A/tLPNi8ATWZ2MoCZlZvZUe7eAWwzs+SkJZdkL3yR/aMSgZQ8d+82s4cJpjOND7LpE8A3CdoIHiNIIP291nuB74fVQVGCuSWWAVcA88ysk8ETjkheaRhqKXnhL/yngfe5+4oBtjkTuNbdz8tnbCL5oKohKWlmdiTBxCYPDpQERIqdSgQiKcxsDkGPnlRdYddQkaKkRCAiUuJUNSQiUuKUCERESpwSgYhIiVMiEBEpcf8fgVX3TNjAv8YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbc7ec26860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_vs_y(df.y_pred, df.y, kind='kde')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T07:44:35.154152Z",
     "start_time": "2018-08-04T07:44:34.983618Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8042890838873323\n"
     ]
    }
   ],
   "source": [
    "print(roc_auc_score(df.y, df.y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T07:44:46.373836Z",
     "start_time": "2018-08-04T07:44:36.163525Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt0VfWd9/H3N3cCBAgJd0JAg4iIoAFvbccblnasulprodNRp64y7dQ61U6rrmfaztinM7bT1tWZ+tShVq0dlVo7VqaFwbZaay1ogqIQEAkBISSYQCAhCbmd833+OBt7jAnZQJKTk/N5rXUW+/LbO9+fwv5k//Y+e5u7IyIikpboAkREZGhQIIiICKBAEBGRgAJBREQABYKIiAQUCCIiAigQREQkoEAQERFAgSAiIoGMRBdwIgoKCry4uDjRZYiIJJWNGzcecPfCvtolVSAUFxdTXl6e6DJERJKKmb0Vpp2GjEREBFAgiIhIQIEgIiKAAkFERAIKBBERARQIIiISUCCIiAiQZN9DEJHhKxJ1OrqitHdFONoZoaU9QmtHF0c7InRFnY5IlEjE6Yo6kajTFY0Sdacr4kTdiUQhEo2+sz7qTtQh6s6xNwUfe2Vw9zcHe9wyx9+z/njeaTrAryO+8aJixo/KHtCfoUAQkVDcnfauKC3tXRxp66I57s/m9k6a27po7YjQ0hHhaEdX8Gfw6Yx92jojtHdG6YxEae8KPp0R2roidEaS//3uZgO376sXTB0agWBmS4HvA+nAA+5+T7f1RcBPgLFBmzvdfU2w7i7gZiAC3Oru68LsU0QGRjTqNLV1crClg0MtHTS0dHC4tZNDrR0cbOmg/kg79UfaaTza+c5v6K2dEVrau0IftHOz0snNSmdEVjq5mRnkZKUzIjON/JFZZGekkZmeRlZ6GtmZaWRnpJOTmU5OZho5melkZ8SWjcxOZ2RWBjmZ6WSmG5kZaWSkGRlpaWSkG2lmZKQZ6WlGWtqfp9MtNp+eZqQZpAVHaYubBuh+7Dazd5aZxeZTTZ+BYGbpwH3AEqAaKDOz1e6+Na7ZPwJPuPsPzWwusAYoDqaXAWcBU4DfmtnsYJu+9ikiJygadeqOtLP7YAt7G1qpO9JOXVMbdUfa2d/UxtuNsemuaM8H9qyMNCaMzqZwdDYFo7LIzcolJzN2cB+Vk8Go7AxGZqUzOieTUTkZjM7OeGd6VHYGI7PTyclIJy0t9Q6mw0GYM4TFQKW7VwGY2SrgGiD+4O1AXjA9BqgJpq8BVrl7O7DLzCqD/RFinyLSg7bOCG8dbGVH3RF2vN3M7oMt1Da2UdfURm1jG+1d0Xe1H52TwYTR2Uwak8MFp41nUl4OBaOyyR+ZxbiRWeTnZjE2N5NxI7MYmZWekr8ZS0yYQJgK7I2brwbO79bmn4BnzOwLwEjgirhtN3Tbdmow3dc+RVJWJOrsrG/mzbePsPtAC7sOtLKnoYW9DUfZ39T2TjszmDp2BJPH5DBv6hiWzJ1I0fiRFI/PpSg/l4l5OeRkpiewJ5JMwgRCT78udD/fXA487O7fNbMLgZ+a2bzjbNvT7a49nsOa2QpgBUBRUVGIckWSh7uzv6mNXfUtVB1oYdeBFipqGtlc3UhLR+SddhPzspmRP5L3lRRQlJ/LjPG5lEwYzazCkTrgS78JEwjVwPS4+Wn8eUjomJuBpQDuvt7McoCCPrbta58E+1sJrAQoLS1N/tsQJCW1dUZ4Y/8Rdh1oZld9C7sOtlJV38yuAy20xh34szPSmDM5j+vOm8Y508cyZ1IexQW55GbphkAZeGH+lpUBJWY2E9hH7CLxJ7u12QNcDjxsZmcCOUA9sBp4zMy+R+yicgnwMrEzh772KZKUjh38N1cf5vXqRjbva2RHXTOR4EJumsHUcSOYVTCKxTPzmVU4ilkFI5lZMJJJeTm6ICsJ02cguHuXmd0CrCN2i+iD7l5hZncD5e6+GvgS8CMzu43Y0M9NHvsGSIWZPUHsYnEX8Hl3jwD0tM8B6J/IgNt3+Chluxp4dc8hXtlzmG21Te/cxZM/Mouzg7H9s6aM4fQJo5ieP4LsDA3zyNBjPsDfrutPpaWlrjemSaLVHD7KS7sOsn7nQTZUNbCnoRWI3Xt/zrSxLCgayznTxjBv6himjh2hu3Yk4cxso7uX9tVOA5MixxGNOpX1zZTvPkT5Ww28vKuB6kNHAcjLyeD8WeO56aJizp+VzxkTR5ORrseDSfJSIIh0U3P4KC/sqOeFHQd4sfIAh1o7ARg/MovS4nF8+uKZnD8rnzmT8kjXeL8MIwoESXmNrZ1s3NPACzsO8Ic369lZ3wLAhNHZXDpnAhfOGk9pcT7F43M1/CPDmgJBUs7Rjggv7Kjnue11lO8+xI66ZiB2y+f5s8azfHER7y8pZPbEUQoASSkKBEkJdU1tPLe9jt9sreOPlfW0dUYZnZPBouJ8rl04lYVFYzm3aJy+5CUpTYEgw9b+xjZWv7aPX71ey+vVjQBMGZPDJ0qns2TuJM6flU+mLgKLvEOBIMNKW2eEtVtqeXJjNX/aeRB3OGfaGL78wTO4bM4E5kwarWEgkV4oEGRYqKhp5ImyvTz16j6a2rooys/l1stKuHbhVGYWjEx0eSJJQYEgSetgcztPvbqPX7yyj221TWSlp/HBeZNYvmg6F8war0dAiJwgBYIkFXfn5V0NPPrSHtZuqaUz4pwzbQx3X3MWH5k/hXEjsxJdokjSUiBIUmho6eC/X6lmVdleKuuaycvJ4FMXzGD54iJmTxyd6PJEhgUFggxZ0aizoeogj728h2cq3qYjEuXcorF8+7r5fGT+FEZk6RZRkf6kQJAh53BrBz8r28vjL+9h98FWxozI5K8uKGLZoiLOmKSzAZGBokCQIWP7/iM8/KddPPXqPto6oywuzueLV8xm6bxJ+sKYyCBQIEhCRaLOs2/U8dCLu/jTzoNkZ6Tx0XOncuNFxcyZlJfo8kRSigJBEmLPwdbgltFq9jS0MmVMDl9ZegbLFxXpTiGRBAkVCGa2FPg+sbebPeDu93Rbfy9waTCbC0xw97Fmdilwb1zTOcAyd/+lmT0M/AXQGKy7yd03nXRPZMhrbO3k15tr+e9Xqil/6xBmcMHM8dyxdA4fPGui3iUgkmB9BoKZpQP3AUuAaqDMzFa7+9Zjbdz9trj2XwAWBsufAxYEy/OBSuCZuN1/2d2f7Id+yBDl7qzfeZBHX9rDb7a9TUdXlNMnjOIrS8/g2gVTmTJ2RKJLFJFAmDOExUClu1cBmNkq4Bpi70nuyXLg6z0svw5Y6+6tJ1OoJJf2rghPb6rhwT/u4o39RxiXm8knFxfxsXOnMW9qnp4nJDIEhQmEqcDeuPlq4PyeGprZDGAm8GwPq5cB3+u27Jtm9jXgd8Cd7t7ewz5XACsAioqKQpQridTWGWHVy3u4//kq9je1MWfSaL593XyuPmeK7hQSGeLCBEJPv8p5L22XAU+6e+RdOzCbDJwNrItbfBewH8gCVgJ3AHe/5we5rwzWU1pa2tvPlQRr64zw6Et7uP/5ndQfaWdxcT7/9vH5vO/0Ap0NiCSJMIFQDUyPm58G1PTSdhnw+R6WXw885e6dxxa4e20w2W5mDwH/EKIWGWKOnRH8v9/vpO5IOxfOGs9/LF/IBbPGJ7o0ETlBYQKhDCgxs5nAPmIH/U92b2RmZwDjgPU97GM5sTOC+PaT3b3WYr8+XgtsOcHaJYGq6ptZVbaXJzdW09DSweKZ+Xx/2UIuPE1BIJKs+gwEd+8ys1uIDfekAw+6e4WZ3Q2Uu/vqoOlyYJW7v2tYx8yKiZ1hPN9t14+aWSGxIalNwGdPpSMyOF7be5jvPLOdF3YcICPNuOLMidxw0QwunDVeQ0MiSc66Hb+HtNLSUi8vL090GSmpqr6Z7zyznTWb95M/Moub3zeTj583jQl5OYkuTUT6YGYb3b20r3b6prIc14Hmdr7/2x089vIesjPSuPXyEj7z/pmMzslMdGki0s8UCNKjzkiUlX+o4oe/38nRzgifXFzErZeXUDg6O9GlicgAUSDIe+w52Mqtq15l097DLJk7kTs/NIfTCkcluiwRGWAKBHmXp16t5qu/rMAMfvDJhVw1f0qiSxKRQaJAEABaO7r46i8r+MUr1SwqHse9n1jAtHG5iS5LRAaRAkHYvv8In3/sFXbWN3Pr5SXcetnpevKoSApSIKS4Zyr2c+uqVxmVncl/3Xw+F59ekOiSRCRBFAgp7Imyvdz5368zf9pYVt5wHhNG6zsFIqlMgZCi7n9+J/esfYMPzC7k/k+dS26W/iqIpDodBVKMu/Ot/93O/c/v5Kr5k/ne9QvIytD1AhFRIKSUSNT56tNbeOylPfzV+UXcfc080tP0/CERiVEgpIjOSJR/+PlrPL2phs9dchpf+eAZehidiLyLAiEFNB7t5JbHXuGFHQf4ytIz+LtLTk90SSIyBCkQhrk9B1v59E/K2H2ghW9/bD7XL5re90YikpIUCMNY2e4G/vanG4lEnUduXsxFp+k7BiLSOwXCMBSNOj96oYpvr9vO9HEj+PFNi/RwOhHpU6hAMLOlwPeJvTHtAXe/p9v6e4FLg9lcYIK7jw3WRYDNwbo97n51sHwmsArIB14B/trdO06tO3K4tYMvPfEav3ujjg+fPYl7PjafPL27QERC6DMQzCwduA9YAlQDZWa22t23Hmvj7rfFtf8CsDBuF0fdfUEPu/4WcK+7rzKz+4GbgR+eXDcEYi+zWbZyA28dbOGfrz6LGy6coTuJRCS0MN9IWgxUuntV8Bv8KuCa47RfDjx+vB1a7Ch1GfBksOgnwLUhapFeHGrp4FMPvET1oVZ+evP53HhRscJARE5ImECYCuyNm68Olr2Hmc0AZgLPxi3OMbNyM9tgZscO+uOBw+7eFWKfK4Lty+vr60OUm3oaj3by1w++RNWBFh64YREXzBqf6JJEJAmFuYbQ06+Z3kvbZcCT7h6JW1bk7jVmNgt41sw2A01h9+nuK4GVAKWlpb393JR1pK2TGx58mTf3N/OfN5zH+0p0J5GInJwwZwjVQPzN69OAml7aLqPbcJG71wR/VgG/J3Z94QAw1syOBdLx9im9aGnv4qaHyqjY18gPPrmQS8+YkOiSRCSJhQmEMqDEzGaaWRaxg/7q7o3M7AxgHLA+btk4M8sOpguAi4Gt7u7Ac8B1QdMbgadPpSOp5mhHhE8/XMamvYf5j+ULufKsSYkuSUSSXJ+BEIzz3wKsA7YBT7h7hZndbWZXxzVdDqwKDvbHnAmUm9lrxALgnri7k+4AbjezSmLXFH586t1JDW2dET7zSDlluxv43vXn8KGzJye6JBEZBuzdx++hrbS01MvLyxNdRkK1d0VY8chG/rCjnu9cdw4fO29aoksSkSHOzDa6e2lf7fQg/CTS0RXl84++wvNv1nPPR89WGIhIv1IgJIlI1Ln18Vf57bY6/u+18/jEoqJElyQiw4wCIQm4O//8PxX8b8V+vnbVXD51wYxElyQiw5ACIQk88MIuHln/Fn/7gVl8+n0zE12OiAxTCoQhbs3mWr65Zht/efZk7lg6J9HliMgwpkAYwl7Zc4jbfraJ82aM47vXn0Oa3n8sIgNIgTBE7Tt8lBWPbGRiXg4/uqGUnMz0RJckIsOcXpAzBDW3d3Hzw2W0d0VYteJ88kdmJbokEUkBCoQhJhJ1/v7xV9lR18xDNy3i9AmjE12SiKQIDRkNIdGoc+cvXud3b9Tx9Y/M5QOzCxNdkoikEAXCEOHufH11BT/fWM3fX17CDRcWJ7okEUkxCoQhwN35lzXb+OmG2HcNvnhFSaJLEpEUpEAYAh56cTc/emEXN11UzJ0fmqNXX4pIQigQEuyVPYf4lzXbuOLMiXztqrkKAxFJGAVCAh1q6eCWR19h0pgcvvtxffFMRBJLt50mSDTq3P7EJg40d/Dk5y5kTG5moksSkRQX6gzBzJaa2XYzqzSzO3tYf6+ZbQo+b5rZ4WD5AjNbb2YVZva6mX0ibpuHzWxX3HYL+q9bQ98Df6ziue31fPWqM5k/bWyiyxER6fsMwczSgfuAJUA1UGZmq+NehYm73xbX/gvAwmC2FbjB3XeY2RRgo5mtc/fDwfovu/uT/dSXpPHG/ia+s+5Nrpw7UY+yFpEhI8wZwmKg0t2r3L0DWAVcc5z2y4HHAdz9TXffEUzXAHVASn/bqr0rwm0/e428ERn860fP1kVkERkywgTCVGBv3Hx1sOw9zGwGMBN4tod1i4EsYGfc4m8GQ0n3mll2L/tcYWblZlZeX18fotyh7fu/3cG22ib+9aPzGT+qxy6LiCREmEDo6VdY76XtMuBJd4+8awdmk4GfAn/j7tFg8V3AHGARkA/c0dMO3X2lu5e6e2lhYXKfXGx8q4H7n9/J9aXTWDJ3YqLLERF5lzCBUA1Mj5ufBtT00nYZwXDRMWaWB/wa+Ed333BsubvXekw78BCxoalhq7Wjiy898RqTx4zgq1fNTXQ5IiLvESYQyoASM5tpZlnEDvqruzcyszOAccD6uGVZwFPAI+7+827tJwd/GnAtsOVkO5EM7ln7BrsPtvKdj5/D6BzdYioiQ0+fdxm5e5eZ3QKsA9KBB929wszuBsrd/Vg4LAdWuXv8cNL1wAeA8WZ2U7DsJnffBDxqZoXEhqQ2AZ/tlx4NQX/ccYBH1r/Fpy+eyYWnjU90OSIiPbJ3H7+HttLSUi8vL090GSekqa2Tpff+gZysdNbc+n69+UxEBp2ZbXT30r7a6ZvKA+yetW+wv6mNX3zuIoWBiAxpepbRANrx9hFWvbyHGy4sZmHRuESXIyJyXAqEAfSt/32DkVkZ3Hq53m8gIkOfAmGAvFR1kN9uq+Ozl5xG/sisRJcjItInBcIAcHf+de0bTMrL4dMXz0x0OSIioSgQBsDaLfvZtPcwty+ZzYgsXUgWkeSgQOhnkajznWe2M3viKD523rRElyMiEpoCoZ/9enMtVfUtfPGK2aTrDWgikkQUCP0oGnV+8OwOSiaMYulZkxJdjojICVEg9KN1Fft58+1mbrnsdL0fWUSSjgKhn0Sjzvd/t4NZBSO5av6URJcjInLCFAj95Lfb3uaN/Ue45bLTde1ARJKSAqEfuDs/eK6SGeNzufocnR2ISHJSIPSDst2HeL26kRUfmEVGuv6Tikhy0tGrH/z4j1WMzc3kowv1vQMRSV6hAsHMlprZdjOrNLM7e1h/r5ltCj5vmtnhuHU3mtmO4HNj3PLzzGxzsM9/D96clnT2HGzlma1v81fnF+lbySKS1Pp8H4KZpQP3AUuIvV+5zMxWu/vWY23c/ba49l8AFgbT+cDXgVLAgY3BtoeAHwIrgA3AGmApsLaf+jVoHvrTLtLN+OsLihNdiojIKQlzhrAYqHT3KnfvAFYB1xyn/XLg8WD6g8Bv3L0hCIHfAEuD9ynnufv64JWbjxB7r3JSOdLWyc/Lq7lq/mQmjclJdDkiIqckTCBMBfbGzVcHy97DzGYAM4Fn+9h2ajDd5z6Hsp+V7aW5vYub3zcr0aWIiJyyMIHQ09h+by9iXgY86e6RPrYNvU8zW2Fm5WZWXl9f32exgyUadX6yfjeLisdx9rQxiS5HROSUhQmEamB63Pw0oKaXtsv483DR8batDqb73Ke7r3T3UncvLSwsDFHu4Hh172H2Nhxl+eKiRJciItIvwgRCGVBiZjPNLIvYQX9190ZmdgYwDlgft3gdcKWZjTOzccCVwDp3rwWOmNkFwd1FNwBPn2JfBtXazbVkpadxxdyJiS5FRKRf9HmXkbt3mdktxA7u6cCD7l5hZncD5e5+LByWA6uCi8THtm0ws28QCxWAu929IZj+HPAwMILY3UVJc4eRu7N2y37eX1JAXk5mossREekXfQYCgLuvIXZraPyyr3Wb/6detn0QeLCH5eXAvLCFDiWvVTey7/BRblsyO9GliIj0G31T+SSs3VxLZrqx5EwNF4nI8KFAOEHuzq8313Lx6QWMydVwkYgMHwqEE7RlXxPVh47y4XmTE12KiEi/UiCcoF9vriUjzbjyLA0XicjwokA4AbG7i2q58LTxjM3NSnQ5IiL9SoFwAipqmnjrYCt/ebaGi0Rk+FEgnID/ea2GjDRj6bxJiS5FRKTfKRBCcnd+9Xot7y8p0HCRiAxLCoSQXtlzmH2Hj3LVfL0zWUSGJwVCSL96vYasjDSW6O4iERmmFAghRKLOr1+v5ZLZhXp2kYgMWwqEEMp2N1B3pJ2PnKPhIhEZvhQIIfzPazWMyEzn8jMnJLoUEZEBo0DoQ1ckytot+7n8zAnkZoV6OKyISFJSIPThxZ0HaWjp0N1FIjLsKRD6sHpTDaOzM7jkjKHz+k4RkYEQKhDMbKmZbTezSjO7s5c215vZVjOrMLPHgmWXmtmmuE+bmV0brHvYzHbFrVvQf93qH22dEZ6p2M/SeZPIyUxPdDkiIgOqz0FxM0sH7gOWANVAmZmtdvetcW1KgLuAi939kJlNAHD354AFQZt8oBJ4Jm73X3b3J/urM/3t99vrONLexdULNFwkIsNfmDOExUClu1e5ewewCrimW5vPAPe5+yEAd6/rYT/XAWvdvfVUCh5Mq1+roWBUFhfOGp/oUkREBlyYQJgK7I2brw6WxZsNzDazF81sg5kt7WE/y4DHuy37ppm9bmb3mll26KoHwZG2Tn63rY6/PHsyGem61CIiw1+YI531sMy7zWcAJcAlwHLgATMb+84OzCYDZwPr4ra5C5gDLALygTt6/OFmK8ys3MzK6+vrQ5TbP56peJv2rqiGi0QkZYQJhGpgetz8NKCmhzZPu3unu+8CthMLiGOuB55y985jC9y91mPagYeIDU29h7uvdPdSdy8tLBy8O31Wv1bD1LEjOLdo3KD9TBGRRAoTCGVAiZnNNLMsYkM/q7u1+SVwKYCZFRAbQqqKW7+cbsNFwVkDZmbAtcCWk+nAQDjY3M4fKw/wkXOmECtPRGT46/MuI3fvMrNbiA33pAMPunuFmd0NlLv76mDdlWa2FYgQu3voIICZFRM7w3i+264fNbNCYkNSm4DP9k+XTt0fdtQTibrejCYiKSXUsxjcfQ2wptuyr8VNO3B78Om+7W7eexEad7/sBGsdNC/vOsTonAzmTslLdCkiIoNGt8/0oGx3A+fNGEd6moaLRCR1KBC6aWjpoLKumUXF+YkuRURkUCkQuinf3QDA4pkKBBFJLQqEbsp2N5CVkcb8aWMSXYqIyKBSIHTz8u5DnDNtDNkZepidiKQWBUKc1o4uKvY16vqBiKQkBUKcTXsO0xV1Fun6gYikIAVCnJd3N2CGHlchIilJgRCnbHcDcyblMWZEZqJLEREZdAqEQGckyqt7DrO4WGcHIpKaFAiBrTVNtHZEKNUFZRFJUQqEwCt7DgFQqjMEEUlRCoTA1pomCkZlMSkvJ9GliIgkhAIhsLW2iTMn5+n9ByKSshQIQEdXlB1vN+tx1yKS0hQIwM76ZjoiUeZOViCISOoKFQhmttTMtptZpZnd2Uub681sq5lVmNljccsjZrYp+KyOWz7TzF4ysx1m9rPg9ZwJsbWmCYCzdIYgIimsz0Aws3TgPuBDwFxguZnN7damBLgLuNjdzwK+GLf6qLsvCD5Xxy3/FnCvu5cAh4CbT60rJ29rbRM5mWnMLBiVqBJERBIuzBnCYqDS3avcvQNYBVzTrc1ngPvc/RCAu9cdb4cWu3J7GfBksOgnwLUnUnh/2lrTxBmT8vSGNBFJaWECYSqwN26+mve+I3k2MNvMXjSzDWa2NG5djpmVB8uPHfTHA4fdves4+xwU7s7W2iZdPxCRlJcRok1PvzZ7D/spAS4BpgEvmNk8dz8MFLl7jZnNAp41s81AU4h9xn642QpgBUBRUVGIck9MTWMbjUc7df1ARFJemDOEamB63Pw0oKaHNk+7e6e77wK2EwsI3L0m+LMK+D2wEDgAjDWzjOPsk2C7le5e6u6lhYWFoTp1Io5dUNYtpyKS6sIEQhlQEtwVlAUsA1Z3a/NL4FIAMysgNoRUZWbjzCw7bvnFwFZ3d+A54Lpg+xuBp0+1Mydja00TZjBn0uhE/HgRkSGjz0AIxvlvAdYB24An3L3CzO42s2N3Da0DDprZVmIH+i+7+0HgTKDczF4Llt/j7luDbe4AbjezSmLXFH7cnx0Lq6KmkZkFI8nNCjN6JiIyfIU6Crr7GmBNt2Vfi5t24PbgE9/mT8DZveyzitgdTAm1tbaJBdPHJroMEZGES+lvKjce7aT60FFdPxARIcUDYVttcEFZt5yKiKR2IOgOIxGRP0vpQNhS00jh6GwmjNY7EEREUjsQ9jVy9tQxiS5DRGRISNlAaO3oorKumXkKBBERIIUDYVvtEaIO83T9QEQESOFA2LKvEYCzp+kMQUQEUjgQNu9rpGBUFpPydEFZRARSOBC27GvkrCljiL2aQUREUjIQ2joj7Khr1h1GIiJxUjIQttU2EYm67jASEYmTkoGgC8oiIu+VooHQxLjcTKaM0QVlEZFjUjIQNu9rZN5UXVAWEYmXcoHQ1hnhzbeP6IKyiEg3oQLBzJaa2XYzqzSzO3tpc72ZbTWzCjN7LFi2wMzWB8teN7NPxLV/2Mx2mdmm4LOgf7p0fNv3H6Er6goEEZFu+nxjmpmlA/cBS4BqoMzMVse9ChMzKwHuAi5290NmNiFY1Qrc4O47zGwKsNHM1rn74WD9l939yf7sUF+21MQuKOsOIxGRdwtzhrAYqHT3KnfvAFYB13Rr8xngPnc/BODudcGfb7r7jmC6BqgDCvur+JNRUdPEmBGZTBs3IpFliIgMOWECYSqwN26+OlgWbzYw28xeNLMNZra0+07MbDGQBeyMW/zNYCjpXjPLPsHaT0plXTMlE0bpgrKISDdhAqGnI6d3m88ASoBLgOXAA2b2zpvrzWwy8FPgb9w9Giy+C5gDLALygTt6/OFmK8ys3MzK6+vrQ5R7fFX1LcwqHHnK+xERGW7CBEI1MD1ufhpQ00Obp9290913AduJBQRmlgf8GvhHd99wbAN3r/WYduAhYkNT7+HuK9291N1LCwtPbbSpqa2TA83tzCodPi0fAAAG00lEQVQcdUr7EREZjsIEQhlQYmYzzSwLWAas7tbml8ClAGZWQGwIqSpo/xTwiLv/PH6D4KwBi43dXAtsOZWOhFFV3wLArAKdIYiIdNfnXUbu3mVmtwDrgHTgQXevMLO7gXJ3Xx2su9LMtgIRYncPHTSzTwEfAMab2U3BLm9y903Ao2ZWSGxIahPw2f7uXHdV9c0AOkMQEelBn4EA4O5rgDXdln0tbtqB24NPfJv/Av6rl31edqLFnqqq+hbS04yi/NzB/tEiIkNeSn1TuepAM0X5uWRlpFS3RURCSakj4866Fl0/EBHpRcoEQiTq7DqoW05FRHqTMoFQc/goHV1RXVAWEelFygTCzuAOo9MUCCIiPUqZQHjnOwgaMhIR6VHqBMKBZvJyMhg/MivRpYiIDEmpEwj1Lcwq1EPtRER6kzKBsLO+WcNFIiLHkRKB0NzexdtN7bqgLCJyHCkRCLv0UDsRkT6lRCBUHdBD7URE+pISgbCzvgUzmDFeD7UTEelNSgRCVX0z08aNICczPdGliIgMWaEef53s5k7JY7oeeS0iclwpEQh/d8npiS5BRGTICzVkZGZLzWy7mVWa2Z29tLnezLaaWYWZPRa3/EYz2xF8boxbfp6ZbQ72+e+mb4yJiCRUn2cIZpYO3AcsAaqBMjNb7e5b49qUAHcBF7v7ITObECzPB74OlAIObAy2PQT8EFgBbCD2NralwNr+7JyIiIQX5gxhMVDp7lXu3gGsAq7p1uYzwH3BgR53rwuWfxD4jbs3BOt+Ayw1s8lAnruvD16/+QhwbT/0R0RETlKYQJgK7I2brw6WxZsNzDazF81sg5kt7WPbqcH08fYpIiKDKMxF5Z7G9r2H/ZQAlwDTgBfMbN5xtg2zz9gPN1tBbGiJoqKiEOWKiMjJCHOGUA1Mj5ufBtT00OZpd+90913AdmIB0du21cH08fYJgLuvdPdSdy8tLCwMUa6IiJyMMIFQBpSY2UwzywKWAau7tfklcCmAmRUQG0KqAtYBV5rZODMbB1wJrHP3WuCImV0Q3F10A/B0v/RIREROSp9DRu7eZWa3EDu4pwMPunuFmd0NlLv7av584N8KRIAvu/tBADP7BrFQAbjb3RuC6c8BDwMjiN1dpDuMREQSyGI3+SQHM6sH3jqBTQqAAwNUzlCVin2G1Ox3KvYZUrPfp9rnGe7e55h7UgXCiTKzcncvTXQdgykV+wyp2e9U7DOkZr8Hq88p8XA7ERHpmwJBRESA4R8IKxNdQAKkYp8hNfudin2G1Oz3oPR5WF9DEBGR8Ib7GYKIiIQ0LAKhr8dzm1m2mf0sWP+SmRUPfpX9K0Sfbw8eR/66mf3OzGYkos7+FuZR7EG768zMzSzp70Y5lcfPJ7MQf8eLzOw5M3s1+Hv+4UTU2Z/M7EEzqzOzLb2st+B1AZVBn8/t1wLcPak/xL4stxOYBWQBrwFzu7X5O+D+YHoZ8LNE1z0Ifb4UyA2mP5fsfQ7b76DdaOAPxB6tXprougfh/3UJ8CowLpifkOi6B6nfK4HPBdNzgd2Jrrsf+v0B4FxgSy/rP0zsS7wGXAC81J8/fzicIYR5PPc1wE+C6SeBy5P8hTx99tndn3P31mB2A+9+dlSyCvP/GuAbwLeBtsEsboCcyuPnk1mYfjuQF0yPoZfnoSUTd/8D0HCcJtcAj3jMBmBs8DqBfjEcAiHM47nfaePuXUAjMH5QqhsYYfoc72aGx6NB+uy3mS0Eprv7rwazsAF0Ko+fT2Zh+v1PwKfMrJrYS7a+MDilJdSJ/ts/IcPhncphHqUd+nHbSeJEHh/+KWJvrPuLAa1ocBy332aWBtwL3DRYBQ2Ck378vLsfHuDaBlKYfi8HHnb375rZhcBPg35HB768hBnQY9lwOEMI+3ju6QBmlkHs9PJ4p2VDXZg+Y2ZXAP8HuNrd2weptoHUV79HA/OA35vZbmJjrKuT/MLyqTx+PpmF6ffNwBMA7r4eyCH2zJ/hLNS//ZM1HAIhzOO5VwM3BtPXAc96cIUmSfXZ52Do5D+JhcFwGFOGPvrt7o3uXuDuxe5eTOzaydXuXp6YcvvFqTx+PpmF6fce4HIAMzuTWCDUD2qVg281cENwt9EFQKPHXifQL5J+yMjDPZ77x8ROJyuJnRksS1zFpy5kn/8NGAX8PLh+vsfdr05Y0f0gZL+HlZB97vXx88kqZL+/BPzIzG4jNmxyU5L/ooeZPU5s6K8guDbydSATwN3vJ3at5MNAJdAK/E2//vwk/+8nIiL9ZDgMGYmISD9QIIiICKBAEBGRgAJBREQABYKIiAQUCCIiAigQREQkoEAQEREA/j9Q2qQqj7gTngAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f175ccc7128>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = []\n",
    "Y = []\n",
    "for i in np.linspace(0,1,100):\n",
    "    df_ = df[df.y_pred<i]\n",
    "    if df_.shape[0]>3:\n",
    "        X.append(i)\n",
    "        Y.append(roc_auc_score(df_.y, df_.y_pred))\n",
    "plt.plot(X, Y); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T07:45:14.520212Z",
     "start_time": "2018-08-04T07:45:13.883190Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8leWd9/HPLztLErawBxIERBZFDOBuq9XiUvEZUaFja62VsQ4zry7TqZ1OO9b6PFM709pOa1t5iuNSq1Tq1KhUWqVqtS6EnbAZKJCQAAlZSMie83v+OEefGAM5gSQn55zv+/XKi3u5Ts7vIsn3XOe673Pf5u6IiEh8SIh0ASIi0ncU+iIicUShLyISRxT6IiJxRKEvIhJHFPoiInFEoS8iEkcU+iIicUShLyISR5IiXUBHI0aM8JycnEiXISISVdavX1/h7lldtet3oZ+Tk0NBQUGkyxARiSpmtj+cdpreERGJIwp9EZE4otAXEYkjCn0RkTii0BcRiSMKfRGROKLQFxGJI2Gdp29mC4AfA4nAL939ex32pwKPA+cBR4Fb3H2fmSUDvwTmhJ7rcXf/9x6sX0SijLvT3BagsSVAQ3Mbx5tbqW9qo7mtjeZWp6UtQGsgQGub0xZwWgPt/w3QFoC2QIDWgBNwCAScgAeXHaf9HWDd//+6f7At2O795bDr7u4DTsHU0elcd/bYXn2OLkPfzBKBh4ArgRJgnZnlu/v2ds3uAKrcfbKZLQYeAG4BbgJS3X2WmQ0EtpvZU+6+r6c7IiI9r7k1QE1DC8caW6htbOVYQwt1Ta3UNbZS29TK8dBXXVMrDc1t1De30dAS/GpuDdDUGqC5tY3GlgCNoe2NLW0EovjW3Ga9972vO3ts5EMfmAcUufteADN7GlgItA/9hcC9oeVVwE/NzAi+OA4ysyRgANAMHOuZ0kUkXO5ObVMrR+uaqahr4lBNI4ePNXKktonaxhbqmtqoez/YG1s41tBKTUMLDS1tXX7v1KQEBqcmMTA1kYHJSaSlJJKalEDGgGRSEhNITU4gLSmRASnBf9OSE0lLTiAtOZGBKUkMSg3+m5qUQHJiAsmJRlJiAkkJRmKCkZxoJCYkkGhGQgIkJyaQmGChdSPBIMGMBLMPAtkMDGu3/P720DZo17YXU7wfCif0xwHF7dZLgPknauPurWZWAwwn+AKwECgDBgJfdvfKjk9gZkuBpQATJkzoZhdE4lsg4FTUNVFa00hpdQMlVfWUVDVQWt1IeV0T5ccaqahrprkt8JHHpiQlkJGWTHpaMHwz0pKZNGIw6WlJZA5IJnNAMhkf/JtEeqjt4NTg16DUJJITdWgwmoQT+p29DHZ8c3aiNvOANmAsMBT4s5m9/P67hg8aui8HlgPk5eVF8Rs/kd7R3BqgpKqe4qoGiivr2VdxnL0Vx9lbXkdJVQOtHeZL0tOSGDdkAFnpqUzOGsGI9BSyBqcyfHAKwwelMiojjdEZaWQMSIq7kW68Cyf0S4DsduvjgdITtCkJTeVkApXAp4GX3L0FOGJmbwJ5wF5E5CPcnaPHm3nvcB3vHaml8OAxth6sYffh2g8Fe2pSArkjBjFjbCZXzxrD2Mw0xmQOYOyQAYwbOoDMAckR7IX0Z+GE/jpgipnlAgeBxQTDvL184DbgLWARsNbd3cwOAJeb2a8ITu+cD/yop4oXiWaBgLP7SC0bD1Szs+wYOw/VsvtwLVX1LR+0GTowmZnjMrnzzElMzhpM9rCBZA8bwKj0NBISNEKX7usy9ENz9MuANQRP2XzE3QvN7D6gwN3zgRXAE2ZWRHCEvzj08IeA/wa2EZwC+m9339IL/RDp19ydsppGCkuPUVhaw9aSGgr2V1HTEAz4QSmJTB2dzidnjGbKqHSmjBzMlFGDGZ2RpukX6VHmvXzeaXfl5eW5rqcv0SwQcPZX1rOjLBTwB4+x7WANlcebgeBZI7kjBjF34jDm5g5jbs5QsocO1MhdTouZrXf3vK7a9bubqIhEm6bWNjYeqObtvUd5e+9RtpbUcLw5eKpjUoIxZVQ6V0wbyazxmcwYm8G00RkMStWfnkSGfvNEuqljyG84UE1zawAzmD4mgxvPG8+MsRlMH5PJlFGDSUtOjHTJIh9Q6It0wd3ZfbiOP+06wptFFazbV0ljSzDkZ4zN4DPnT2R+7jDm5w4nc6DOmpH+TaEv0omWtgBv7z3KHwoPs3bnEQ5WNwAwddRgFs+dwEWTRzAvd5hOjZSoo9AXCampb+H198pZu/MIr+w4zLHGVgYkJ3LxlBEsu3wyHz9zJKMz0yJdpshpUehLXDtU08iLW8t4aVsZ6/dXEXAYMjCZK6eP5pMzRnHp1CzNyUtMUehL3Kmoa+L32w7x/OZS1u2rxB3OGpPB3398Mh87cySzs4eQqNMnJUYp9CUuVB5v5g+Fh3hxaxl/2XOUtoAzeeRgvvyJqVx39hgmZQ2OdIkifUKhLzGrtrGFF7eUkb+5lHf+WklbwJkwbCB3XTaJT50zljNHpevTrhJ3FPoSUwIB5+29R/lNQTEvFR6isSXApBGD+OJlZ7Bg5mhmjM1Q0EtcU+hLTDhU08gzBcX8Zn0xxZUNpKclceOc8dyUl8054zMV9CIhCn2JWu7OhgNVPPLmPl7adoi2gHPBpOF89cozWTBztM66EemEQl+iTiDg/GH7YX7+ahGbS2pIT0vi8xflcOv5E5k4fFCkyxPp1xT6EjVa2wK8sKWMn71axO7DdUwYNpDvLpzB38wZrwuYiYRJfynS7zW2tPHM+hKWv76H4soGpo4azI8Xz+baWWNI0v1ZRbpFoS/9VkNzG0++s5+HX99LeW0Ts7OH8K1rp/OJs0bp2vMip0ihL/1OQ3Mbv3p7Pw+/voeKumYumDScHy+ezQWThussHJHTFFbom9kC4McEb5f4S3f/Xof9qcDjwHnAUeAWd99nZn8LfK1d07OBOe6+qSeKl9jy/sj+F68Fw/6iycP52RVTmZc7LNKlicSMLkPfzBIJ3uv2SqAEWGdm+e6+vV2zO4Aqd59sZouBBwgG/5PAk6HvMwt4ToEvHTW2tPHUuwf42at7KK9t4qLJw/n5J6YyN0dhL9LTwhnpzwOK3H0vgJk9DSwE2of+QuDe0PIq4KdmZv7hG/AuAZ467YolZrS0BXimoISfrH2PsppG5ucO46dLzmX+pOGRLk0kZoUT+uOA4nbrJcD8E7Vx91YzqwGGAxXt2txC8MVB4py789K2Q3x/zS7+WnGccycM4T9vOocLz9CcvUhvCyf0O/sr9O60MbP5QL27b+v0CcyWAksBJkyYEEZJEq3W76/k/hd3sPFANVNHDWbFbXlcPm2kwl6kj4QT+iVAdrv18UDpCdqUmFkSkAlUttu/mJNM7bj7cmA5QF5eXscXFIkBZTUNfO/3O3luUykj01N54MZZLDovW9etF+lj4YT+OmCKmeUCBwkG+Kc7tMkHbgPeAhYBa9+fzzezBOAm4NKeKlqiR31zK8tf38vDr+0l4M4/Xj6Zuz52BgNTdLawSCR0+ZcXmqNfBqwheMrmI+5eaGb3AQXung+sAJ4wsyKCI/zF7b7FpUDJ+weCJT60BZxV64v5wR92c6S2iWtnjeGeq6eRPWxgpEsTiWv24RNsIi8vL88LCgoiXYachtd2l/Pvq3ew81AtcyYM4ZvXnsV5E3X6pUhvMrP17p7XVTu9x5Yes6PsGP9n9Q7+/F4FE4YN5KFPz+GaWaN1kFakH1Hoy2mrrm/mh3/cza/e3k96WjLfum46t54/gdQkXc9epL9R6Mspc3dWrivmgZd2UtPQwmfOn8iXr5zKkIEpkS5NRE5AoS+npLS6ga//dgt/fq+CeTnDuPf6GUwfmxHpskSkCwp96RZ359kNB7n3+ULaAs79N8zkb+dP0Ly9SJRQ6EvYKuqa+Jdnt/KH7YeZmzOU/7zpHN2eUCTKKPQlLC9tO8Q3/2crtU2tfPOas/j8xbn6NK1IFFLoy0mV1zZx7/OFvLiljJnjMnjq5tlMHZUe6bJE5BQp9KVT78/df/fF7dQ3tfHVK6dy18fOIFn3pBWJagp9+Yjaxha+/tstrN56iPMmDuWBG2cxeaRG9yKxQKEvH7Kj7Bh3P7mBA5X1fH3BNP7u0km6CblIDFHoywee3VDCN57dSuaAZH79hfm6g5VIDFLoCy1tAf73izt49C/7OH/SMH6yZA5Z6amRLktEeoFCP86V1zbx909u4N19lXzh4lzuuXoaSTpYKxKzFPpxbG95HZ995F0q6pr48eLZLJw9LtIliUgvU+jHqQ0Hqrjj0XUkmLFy6QWckz0k0iWJSB9Q6MehtTsPc/eTGxiVkcZjt88jZ4QupSASL8KavDWzBWa2y8yKzOyeTvanmtnK0P53zCyn3b6zzewtMys0s61mltZz5Ut3/X5rGUsfX8+Uken89osXKvBF4kyXoW9micBDwNXAdGCJmU3v0OwOoMrdJwMPAg+EHpsE/Aq4y91nAB8DWnqseumW5zYdZNlTGzknewhP3jmfEYN1ho5IvAlnpD8PKHL3ve7eDDwNLOzQZiHwWGh5FXCFBa+1exWwxd03A7j7UXdv65nSpTtWrS/hSys3kTdxKI9/fh4ZacmRLklEIiCc0B8HFLdbLwlt67SNu7cCNcBwYCrgZrbGzDaY2T+ffsnSXc9vLuVrqzZz4RnDefT2eQxK1aEckXgVzl9/Z5/B9zDbJAEXA3OBeuCV0B3bX/nQg82WAksBJkyYEEZJEq6Xtx/myys3MXfiMH752bkMSNF9a0XiWTgj/RIgu936eKD0RG1C8/iZQGVo+2vuXuHu9cBqYE7HJ3D35e6e5+55WVlZ3e+FdOrNogru/vUGpo/NYMXn8hT4IhJW6K8DpphZrpmlAIuB/A5t8oHbQsuLgLXu7sAa4GwzGxh6MbgM2N4zpcvJbDxQxZ2PF5A7fBCP3T6PdM3hiwhhTO+4e6uZLSMY4InAI+5eaGb3AQXung+sAJ4wsyKCI/zFocdWmdkPCb5wOLDa3V/spb5ISNGRWm5/dB0jBqfyxB3zGDooJdIliUg/YcEBef+Rl5fnBQUFkS4jah2sbmDRz/9CS5vz2y9eoHvYisSJ0PHSvK7a6cpaMaSironPrHiHuqZWHv/8PAW+iHyEQj9G1NS38JkV71Ja3cCK2+YyfWxGpEsSkX5IoR8Djje1cvuj71J0pJaHP5PHvNxhkS5JRPophX6Ua2xpY+kTBWwqruYnS87lsqk65VVETkwfzYxiza0B7n5yA28WHeUHN53DgpljIl2SiPRzGulHqda2AF9auZG1O49w/w0zufG88ZEuSUSigEI/CgUCzj+v2sLqrYf412vP4tbzJ0a6JBGJEgr9KPSjl3fz7MaDfPXKqXzhkkmRLkdEoohCP8q8uKWM/1pbxM1541l2+eRIlyMiUUahH0UKS2v4p2c2c97EoXz3hpkEb1kgIhI+hX6UOFrXxNLH1zNkYDI/v3UOqUm6YqaIdJ9O2YwCLW0BvvjkBirqmnjmrgsYma7bDIvIqVHoR4HvvrCdd/9ayY9umc3Z44dEuhwRiWKa3unnVq47wONv7efOS3K54dyOd6kUEekehX4/tuFAFf/6u21cMmUEX18wLdLliEgMUOj3UxV1Tdz9qw2MzkzjJ0vOJSlRPyoROX2a0++HWtsC/ONTG6mqb+bZuy9kyEDd+UpEeoZCvx/64R9385c9R/mPRWczY2xmpMsRkRgS1pyBmS0ws11mVmRm93SyP9XMVob2v2NmOaHtOWbWYGabQl+/6NnyY8/L2w/zs1f3sGReNjflZUe6HBGJMV2O9M0sEXgIuBIoAdaZWb67b2/X7A6gyt0nm9li4AHgltC+Pe4+u4frjkmHjzXytVWbmTE2g3/71IxIlyMiMSickf48oMjd97p7M/A0sLBDm4XAY6HlVcAVpmsEdEsg4Hz1N5tpaGnjv5acS1qyPnErIj0vnNAfBxS3Wy8Jbeu0jbu3AjXA8NC+XDPbaGavmdklnT2BmS01swIzKygvL+9WB2LFijf+yhtFFXz7uhmckTU40uWISIwKJ/Q7G7F7mG3KgAnufi7wFeDXZvaRO3a7+3J3z3P3vKys+Lvd37aDNXx/zU4+OWMUS+ZpHl9Eek84oV8CtE+i8UDpidqYWRKQCVS6e5O7HwVw9/XAHmDq6RYdSxpb2vjSyk0MHZjC9/7mbF05U0R6VTihvw6YYma5ZpYCLAbyO7TJB24LLS8C1rq7m1lW6EAwZjYJmALs7ZnSY8P3fr+ToiN1/ODmcxg6SOfji0jv6vLsHXdvNbNlwBogEXjE3QvN7D6gwN3zgRXAE2ZWBFQSfGEAuBS4z8xagTbgLnev7I2ORKM/v1fOo3/Zx+cuzOGSKfE3rSUifc/cO07PR1ZeXp4XFBREuoxeV13fzCd/9Drpacm88A8X62wdETktZrbe3fO6aqcLukSAu/PN323jaF0zP7pltgJfRPqMQj8CfrvhIC9uKePLV05l5jhdZkFE+o5Cv4/tqzjOvz23jfm5w7jrsjMiXY6IxBmFfh9qaQvwpZWbSEwwHrxlNokJOj1TRPqWrrLZh37yyntsKq7moU/PYeyQAZEuR0TikEb6fWR76TEeenUPN84Zz7Vnj4l0OSISpxT6faAt4Hzj2S0MHZjMt6+bHulyRCSOKfT7wONv7WNzSQ3fum46mQOTI12OiMQxhX4vK61u4D/X7OKyqVlcf87YSJcjInFOod/Lvv1cIW3u3H/DTF1MTUQiTqHfi17ZcZiXdxzmS5+YSvawgZEuR0REod9bGlva+M7z2zkjaxCfvyg30uWIiAA6T7/XPPzaXg5U1vPkF+aTkqTXVhHpH5RGvaC4sp6fvVrEtbPGcNHkEZEuR0TkAwr9XnDfC9tJMOOb154V6VJERD5Eod/D/vxeOX/cfphll0/WpRZEpN9R6Peg1rYA97+wgwnDBvKFS3TwVkT6n7BC38wWmNkuMysys3s62Z9qZitD+98xs5wO+yeYWZ2Z/VPPlN0/PbWumF2Ha/mXa6aRmqQbo4hI/9Nl6IdubP4QcDUwHVhiZh0vIHMHUOXuk4EHgQc67H8Q+P3pl9t/1TS08MM/7GJ+7jA+OWN0pMsREelUOCP9eUCRu+9192bgaWBhhzYLgcdCy6uAKyz08VMzuwHYCxT2TMn9009eeY/qhha+dd10ffJWRPqtcEJ/HFDcbr0ktK3TNu7eCtQAw81sEPB14DunX2r/tf/ocR57ax83n5et2x+KSL8WTuh3Nmz1MNt8B3jQ3etO+gRmS82swMwKysvLwyipf/nxK++RYMZXrpoa6VJERE4qnE/klgDZ7dbHA6UnaFNiZklAJlAJzAcWmdn3gSFAwMwa3f2n7R/s7suB5QB5eXkdX1D6taIjdfxu40HuuDiXURlpkS5HROSkwgn9dcAUM8sFDgKLgU93aJMP3Aa8BSwC1rq7A5e838DM7gXqOgZ+tPvRy7tJS07UTc5FJCp0Gfru3mpmy4A1QCLwiLsXmtl9QIG75wMrgCfMrIjgCH9xbxbdX+woO8YLW8pY9vHJDB+cGulyRES6FNYF19x9NbC6w7Zvt1tuBG7q4nvcewr19Ws//ONu0tOSuPOSSZEuRUQkLPpE7inaWlLDH7cf5s5LJukWiCISNRT6p+inf3qPjLQkbr8oJ9KliIiETaF/CnYfrmVN4WE+d2EO6Wka5YtI9FDon4Kf/amIgSmJ3K47YolIlFHod9P+o8fJ31zK386fwNBBKZEuR0SkWxT63fSL1/aQlJigM3ZEJCop9LuhrKaBVetLuDlvPCP16VsRiUIK/W549M19tAWcv7tUn74Vkeik0A9TXVMrv373AFfPGkP2sIGRLkdE5JQo9MP0TEExtY2tfOFinbEjItFLoR+GtoDz32/u47yJQzl3wtBIlyMicsoU+mH44/bDHKis1yhfRKKeQj8MK97Yy/ihA7hK974VkSin0O/C5uJq1u2r4vaLcklM0L1vRSS6KfS78Phb+xmUksjNeeMjXYqIyGlT6J9ETX0LL2wp5YZzx+nCaiISExT6J/G7TQdpag2wZN6ESJciItIjwgp9M1tgZrvMrMjM7ulkf6qZrQztf8fMckLb55nZptDXZjP7Xz1bfu9xd5569wBnj89k5rjMSJcjItIjugx9M0sEHgKuBqYDS8xseodmdwBV7j4ZeBB4ILR9G5Dn7rOBBcDDZhbWLRojbWNxNTsP1WqULyIxJZyR/jygyN33unsz8DSwsEObhcBjoeVVwBVmZu5e7+6toe1pgPdE0X3hqXcOMCglkU+dMzbSpYiI9JhwQn8cUNxuvSS0rdM2oZCvAYYDmNl8MysEtgJ3tXsR6LeONbbw/JZSrp89jsGpUfHGREQkLOGEfmcnp3ccsZ+wjbu/4+4zgLnAN8zsI9ckNrOlZlZgZgXl5eVhlNS7ntt4kMaWAJ/W1I6IxJhwQr8EyG63Ph4oPVGb0Jx9JlDZvoG77wCOAzM7PoG7L3f3PHfPy8rKCr/6XuDu/PrdYmaMzWDWeB3AFZHYEk7orwOmmFmumaUAi4H8Dm3ygdtCy4uAte7uocckAZjZROBMYF+PVN5Lth08xo6yYyzWKF9EYlCXE9bu3mpmy4A1QCLwiLsXmtl9QIG75wMrgCfMrIjgCH9x6OEXA/eYWQsQAO5294re6EhPeXrdAdKSE7heB3BFJAaFdZTS3VcDqzts+3a75Ubgpk4e9wTwxGnW2Gfqm1vJ31TKNbPGkDlAn8AVkdijT+S2s3rrIWqbWlk8V1M7IhKbFPrtrFx3gEkjBjE3RzdKEZHYpNAPKTpSy7p9VdwyNxszXUJZRGKTQj9k5bpikhKMv5mjSyiLSOxS6AMtbQH+Z2Mpl08bSVZ6aqTLERHpNQp94PXd5VTUNbHoPI3yRSS2KfSB324oYfigFD4+bWSkSxER6VVxH/rV9c28vP0I188eS3Ji3P93iEiMi/uUe35zKc1tAU3tiEhciPvQX7W+hGmj05kxVhdXE5HYF9eh/97hWjaX1GiULyJxI65Df9WGEhITjIWzO94TRkQkNsVt6Ls7z28q5bKpWTo3X0TiRtyG/taDNZTWNHL1zNGRLkVEpM/Ebei/tO0QiQnGJ84aFelSRET6TNyG/prCQ5w/aRhDB6VEuhQRkT4Tl6FfdKSWPeXHWTBDUzsiEl/iMvRf2nYIgKsU+iISZ8IKfTNbYGa7zKzIzO7pZH+qma0M7X/HzHJC2680s/VmtjX07+U9W/6peanwEOdOGMKojLRIlyIi0qe6DH0zSwQeAq4GpgNLzGx6h2Z3AFXuPhl4EHggtL0C+JS7zwJuox/cL7ekqp5tB49pakdE4lI4I/15QJG773X3ZuBpYGGHNguBx0LLq4ArzMzcfaO7l4a2FwJpZhbRk+LXFB4G4JMKfRGJQ+GE/jiguN16SWhbp23cvRWoAYZ3aHMjsNHdm06t1J6xZtshpo1OJ2fEoEiWISISEeGEfmc3jPXutDGzGQSnfP6u0ycwW2pmBWZWUF5eHkZJp6bqeDMF+yu5arrOzReR+BRO6JcA2e3WxwOlJ2pjZklAJlAZWh8P/A/wWXff09kTuPtyd89z97ysrKzu9aAbXn+vnICjm6WISNwKJ/TXAVPMLNfMUoDFQH6HNvkED9QCLALWurub2RDgReAb7v5mTxV9qtbuPMLwQSmcM35IpEsREYmILkM/NEe/DFgD7AB+4+6FZnafmV0farYCGG5mRcBXgPdP61wGTAa+ZWabQl8RGWa3BZzXdpdz2ZlZJCR0NhslIhL7ksJp5O6rgdUdtn273XIjcFMnj7sfuP80a+wRm4qrqK5v4XJN7YhIHIubT+Su3XmExATjkim9d8xARKS/i6PQL+e8iUPJHJAc6VJERCImLkK/rKaBHWXHNLUjInEvLkL/1V3Bc/8V+iIS7+Ii9NfuPMK4IQOYMnJwpEsREYmomA/95tYAbxZV8PFpWZjpVE0RiW8xH/obD1RR39zGpTprR0Qk9kP/jaIKEgzOP6Pj9d9EROJPXIT+OdlDyEjTqZoiIjEd+scaW9hcXM3Fk0dEuhQRkX4hpkP/7T1HCThcpNAXEQFiPPTfLKpgQHIi507QVTVFRCDGQ/+Nogrm5Q4jNSkx0qWIiPQLMRv6ZTUN7Ck/rvl8EZF2Yjb03yw6Cmg+X0SkvRgO/QqGD0ph2uj0SJciItJvxGTouztvFFVw4eQRukuWiEg7YYW+mS0ws11mVmRm93SyP9XMVob2v2NmOaHtw83sT2ZWZ2Y/7dnST2xvxXHKa5u4UJ/CFRH5kC5D38wSgYeAq4HpwBIzm96h2R1AlbtPBh4EHghtbwS+BfxTj1Uchk0HqgGYM2FoXz6tiEi/F85Ifx5Q5O573b0ZeBpY2KHNQuCx0PIq4AozM3c/7u5vEAz/PrO5pJpBKYlM1qWURUQ+JJzQHwcUt1svCW3rtI27twI1QMTmVjYVVzNrfCaJms8XEfmQcEK/s+T0U2hz4icwW2pmBWZWUF5eHu7DOtXY0saOsmPMztbUjohIR+GEfgmQ3W59PFB6ojZmlgRkApXhFuHuy909z93zsrJO77r328uO0dLmzM7WpRdERDoKJ/TXAVPMLNfMUoDFQH6HNvnAbaHlRcBadw97pN+T3j+Iq9AXEfmopK4auHurmS0D1gCJwCPuXmhm9wEF7p4PrACeMLMigiP8xe8/3sz2ARlAipndAFzl7tt7vitBm0uqGZ2RxujMtN56ChGRqNVl6AO4+2pgdYdt32633AjcdILH5pxGfd22qbiac7Iz+/IpRUSiRkx9IrfyeDP7j9brIK6IyAnEVOhvLtF8vojIycRU6G86UI0ZzBqv6R0Rkc7EVOhvLqlm6sh0BqeGdahCRCTuxEzouzubdRBXROSkYib09x+tp6q+RQdxRUROImZCvzUQ4OqZo5mbo9AXETmRmJn8njwynZ/fel6kyxAR6ddiZqQvIiJdU+iLiMQRhb6ISBxR6IuIxBGFvohIHFHoi4jEEYW+iEgcUeiLiMQRi9BdDU/IzMqB/d14yAigopfK6c/isd/x2GeIz37HY5/h9Po90d2mnQOVAAAD7ElEQVS7vMl4vwv97jKzAnfPi3QdfS0e+x2PfYb47Hc89hn6pt+a3hERiSMKfRGROBILob880gVESDz2Ox77DPHZ73jsM/RBv6N+Tl9ERMIXCyN9EREJU9SEvpktMLNdZlZkZvd0sj/VzFaG9r9jZjl9X2XPC6PfXzGz7Wa2xcxeMbOJkaizJ3XV53btFpmZm1lMnOURTr/N7ObQz7vQzH7d1zX2tDB+vyeY2Z/MbGPod/yaSNTZk8zsETM7YmbbTrDfzOy/Qv8nW8xsTo8W4O79/gtIBPYAk4AUYDMwvUObu4FfhJYXAysjXXcf9fvjwMDQ8hejvd/h9DnULh14HXgbyIt03X30s54CbASGhtZHRrruPujzcuCLoeXpwL5I190D/b4UmANsO8H+a4DfAwacD7zTk88fLSP9eUCRu+9192bgaWBhhzYLgcdCy6uAK8zM+rDG3tBlv939T+5eH1p9GxjfxzX2tHB+1gDfBb4PNPZlcb0onH7fCTzk7lUA7n6kj2vsaeH02YGM0HImUNqH9fUKd38dqDxJk4XA4x70NjDEzMb01PNHS+iPA4rbrZeEtnXaxt1bgRpgeJ9U13vC6Xd7dxAcIUSzLvtsZucC2e7+Ql8W1svC+VlPBaaa2Ztm9raZLeiz6npHOH2+F7jVzEqA1cA/9E1pEdXdv/tuiZZ75HY2Yu942lE4baJN2H0ys1uBPOCyXq2o9520z2aWADwIfK6vCuoj4fyskwhO8XyM4Du6P5vZTHev7uXaeks4fV4CPOruPzCzC4AnQn0O9H55EdOrWRYtI/0SILvd+ng++jbvgzZmlkTwreDJ3kJFg3D6jZl9AvgmcL27N/VRbb2lqz6nAzOBV81sH8E5z/wYOJgb7u/4c+7e4u5/BXYRfBGIVuH0+Q7gNwDu/haQRvD6NLEsrL/7UxUtob8OmGJmuWaWQvBAbX6HNvnAbaHlRcBaDx0ViWJd9js01fEwwcCP9jle6KLP7l7j7iPcPcfdcwgex7je3QsiU26PCed3/HcED9xjZiMITvfs7dMqe1Y4fT4AXAFgZmcRDP3yPq2y7+UDnw2dxXM+UOPuZT31zaNiesfdW81sGbCG4BH/R9y90MzuAwrcPR9YQfCtXxHBEf7iyFXcM8Ls938Ag4FnQsetD7j79REr+jSF2eeYE2a/1wBXmdl2oA34mrsfjVzVpyfMPn8V+L9m9mWCUxyfi/bBnJk9RXCKbkToWMW/AckA7v4LgscurgGKgHrg9h59/ij//xMRkW6IlukdERHpAQp9EZE4otAXEYkjCn0RkTii0BcRiSMKfRGROKLQFxGJIwp9EZE48v8ALoD9hJWYWnUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f175cd217b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = []\n",
    "Y = []\n",
    "for i in np.linspace(0,1,100):\n",
    "    df_ = df[df.y_pred<i]\n",
    "    if df_.shape[0]>3:\n",
    "        X.append(i)\n",
    "        Y.append(df_.y.mean())\n",
    "plt.plot(X, Y); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T07:45:58.078436Z",
     "start_time": "2018-08-04T07:45:57.842984Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VNX9//HXJzuQsO9JIKyyrwERqljFikvBqlXEvVZwobbWLmr7pRZtfy5fv1ZbquKuFXGpVVQU61YVWRI2gbCFEEjYEhIIWcg2Ob8/Em0MgQyQ5E4m7+fjweMxd+bMnQ+X5M2dc889x5xziIhIcAnxugAREal/CncRkSCkcBcRCUIKdxGRIKRwFxEJQgp3EZEgpHAXEQlCCncRkSCkcBcRCUJhXn1wx44dXUJCglcfLyLSJK1cuXK/c65TXe08C/eEhASSk5O9+ngRkSbJzHb4007dMiIiQUjhLiIShBTuIiJBSOEuIhKEFO4iIkFI4S4iEoQU7iIiQUjhLiLih6T0XNZl5nldht8U7iIifvjFgjX8/u31XpfhN8/uUBURaSoyDxSx6+BhsvNLKCn3ERkW6nVJddKZu4hIHZLScwEo9VWwcU++x9X4R+EuIlKHFdtziQirjMs1Ow94XI1/FO4iInVYvj2XCX060DkmkjUZB70uxy/qcxcROYbs/BLSsgu5LDGe8NCQJhPuOnMXETmGb/rbx/Zqz4gebUnPKeJAYanHVdVN4S4icgwrtufSIjyUId3bMCK+LQBrMgP/7F3hLiLNQkWF442VmZz+4Cc88+V2v9+3fHsuo3q2JSIshGFxbTGDtU2ga8avPnczmww8CoQCTzvn7q/xeg/gBaBtVZs7nXOL6rlWEZE6ZeUXM/VvS2gdFc5ZAztz9oDOOOC+d1NYm5lHeKjx7JfbuX58AiEhdsx95RWVsWnvIX5xdn8AoiPD6Nc5ukn0u9cZ7mYWCswFzgEygSQzW+icS6nW7PfAa865x81sELAISGiAekVEjmn2WxvIKSylZ4eWPPV5Go9/tg2ALq0jeeTy4YSY8fMFa1iWlsP4vh2Pua/kHbk4V9nf/o0R8W35d8o+nHOYHfs/By/5c+Y+Fkh1zqUBmNkCYCpQPdwd0LrqcRtgd30WKSLij0Xr9vDBhr38dvIAbj6zD3mHy/hiazY5BaVcOjqOVpFhFJf5iIkM442VmXWG+4rtuYSHGiN7tP32uRHx7XgtOZMdOUUkdGzV0H+lE+ZPn3sskFFtO7PqueruAa4ys0wqz9p/Vi/ViYgcRcruQxSUlH+7faCwlNlvr2dobBtuPL0XAG1ahHPhsO5cOz6BVpGV57JR4aFcOLw776/f+53312b59lyGx7UlKvy/0w18e1E1wLtm/An32r53uBrbVwDPO+figPOBl8zsiH2b2QwzSzaz5Ozs7OOvVkQE+DrzIOc/9gUT7v+Ev3y0hYNFpcx5N4WDRWU8cMkwwkKPHW2Xjo7lcJmPRev2HLVNUWk563flfadLBqB/l2hahIcGRbhnAvHVtuM4stvlBuA1AOfcUiAKOOL7jnNunnMu0TmX2KlTpxOrWESavXmfpxETGcaYhPb85aOtnPb/PuFfq3dxy5l9GNS9dZ3vH9WjHb06tuKfKzOP2mZZWg7lFe6IcA8LDWFobJugCPckoJ+Z9TKzCGAasLBGm53A2QBmNpDKcNepuYjUu4zcIhat28P0U3vw9LWJfPCL0zlnUBcm9u/ErWf19WsfZsYlo2JZvj2XjNyiI173VTge/nAL3dtEMa53hyNeH9GjLSm7D1FS7jvpv09DqTPcnXPlwCxgMbCRylExG8xsjplNqWp2B3Cjma0FXgGuc87V7LoRETlpz3y5nRAzrpuQAMCArq157IqRvPCTscc1Fe+PRsVhBv9cdeTZ++vJGWzYfYi7zh/4nf72b4yIb0upr4JNATxDpF/j3KvGrC+q8dzsao9TgAn1W5qIyHcdLCrl1aQMpozoTrc2LU5qX7FtWzC+TwfeWJnJTRP7fBvih4rLeGjxZsYktOPCYd1qfe/AbpVdP5v35TM8vm2tbbymO1RFpMl4eflODpf5mHFG73rZ37WnJZB54DCXPbmUXQcPA/C3T1LJLSpl9oWDjzqOPb5dCyJCQ9iWVVAvdTQEhbuINAnFZT6eW5LOGf07MaBr3RdN/fGDwV158urRbM8u5MLHvmD+8p08t2Q7l42OZ2hcm6O+Lyw0hF4dW5GqcBcROTlvrd7F/oISZtbTWfs3zh3clbdnTaBTTCR3/2sdkWGh/OrcU+p8X9/O0aRmB264az53EQl4WfnFPLh4M8Pj2zK+z5GjV05W707RvHXrBB5avJlRPdrRKSayzvf06RzN++v3UFzmq/Wiq9cU7iIS0Jxz/OaNryksKefhHw9rsPlcWkaE8YcfDva7fd/O0VQ42L6/8NsLrIFE3TIiEtBeWraDzzZn87sLBtK3c4zX5XyrX+doALYGaL+7wl1EAlZqVj5/em8jE/t34upxPb0u5zt6dWxFiBGwF1UV7iISkDbtPcRtr6yhVWQYDzVgd8yJigoPJb59y+MaDllUWs5ZD3/GO2sbfuJc9bmLSMDIzi/hn6syeWv1LjbtzSc81HjiqtF0jonyurRa9e0UfVxn7tuyCknLLiSsjkVC6oPCXUQCwvpdeVz/fBLZ+SWM7NGWOVMHc8HQbnSIrnvkilf6do7mi637KfdV1DkTJcDWrMrpCvp1afhrBwp3EfHcf7Zkc8s/VtK2ZQSLbjvdr5kdA0GfztGU+irIOHCYXn4s3LE1q4DwUKNnh5YNXpv63EXEU68nZ/CT55Po0aEVb94yvskEO/x3xIy/XTNb9+XTq2Mrwv04yz9ZCncR8cwbKzP59RtfM75PB16bOY4urQOzb/1o+hxvuGcV0K+RhnMq3EXEE8npudz95jom9O3AM9eOISYq3OuSjlvrqHC6tI78ti/9WIrLfOzMLaJfl+hGqEzhLiIeyMgtYuZLK4lt14K/Tx9NRFjTjaK+naP9Gg6ZmlWAc+jMXUSCU0FJOTe+mEyZr4Knr02kTcumd8ZeXd9O0WzLLqSu9Ym+6brRmbuIBB1fheMXC1azNauAuVeOok+nxgm6htS3czQFJeXsPVR8zHZbs/IJCzESOtQ9qqY++BXuZjbZzDabWaqZ3VnL64+Y2ZqqP1vMLLBXjhURT9z3XgofbcziDz8cxOn9OnldTr34Zr6bui6qbtlXQELHVo3WBVXnp5hZKDAXOA8YBFxhZoOqt3HO3e6cG+GcGwH8FXizIYoVkabr+SXbeW5JOjd8rxfXnJbgdTn1pq+fI2ZSswq+HTrZGPz5L2QskOqcS3POlQILgKnHaH8FlYtki4gA8FHKPua8m8I5g7pw9/kDvS6nXnWMjqBNi/BjhntxmY8dOYWNGu7+3KEaC2RU284ETq2toZn1BHoBn5x8aSLS1Djn2JpVwKebssg4UER+cTn5xeUs3ZbDkNg2PDptBKGNMK9KYzIz+naOPubUv2nZhVS4xpl24Bv+hHtt/xJHuyw8DXjDOeerdUdmM4AZAD169PCrQBEJfLsPHuapL9L4aOM+MnIrF5pu1zKc1i3CiYkK44z+Hbn3oiG0jAjOGU9O6RrDe1/vwTlX6+yV/51TJrDO3DOB+GrbccDR5qucBtx6tB055+YB8wASExOPPW5IRJqEz7dk8/MFqyks9TGhTwdumtiHswd0oWubpnW36ckY2K0185fvZE9eMd3btjji9dSsAkJDzK/5Z+qLP+GeBPQzs17ALioDfHrNRmZ2CtAOWFqvFYpIQKqocDz2yVYe/Xgrp3SJ4e9XjqJ3EAxtPBGDulV2t6TsPlRruG/Zl0/PDi2JDGu8tVbrDHfnXLmZzQIWA6HAs865DWY2B0h2zi2sanoFsMDVNZJfRJq0wpJy/p2yj/nLd7IiPZeLR8Xyp4uG0iIi8BaJbiyndK2c7GzjnkNMGtTliNe3NvJIGfBzyl/n3CJgUY3nZtfYvqf+yhKRQLNyRy7PLknn4437KC6roHubKO6/eCiXj4kPuFWSGlt0ZBg92rdk495DR7xWUu5jR04R5w/p1qg1BefVDRE5Iat3HmBZWi5jEtoxPL4t4aEhpGYV8OAHm/gwZR/tWobz49HxTBnRndE92hESZCNfTsbAbjFs3HPkBGLb9xfiq3CNejEVFO4iUuWVFTv5n7fWU15R2bMaHRnGoO6tWbnjAC3CQ7njnP7ccHqvoB3xcrIGdmvNhyn7KCot/84x2rqvak6ZRpow7Bv6VxJp5sp9Fdz33kae/yqdM/p34k8XDWH9rjy+TN3Pyh0HuHpcT352Vt+AXu4uEAzs1hrnYNPefEb1aPft81uzCggx6N2p8UbKgMJdpFk7VFzGrS+v4out+7nhe72467wBhIWGEN++JecNbdw+4qZuULf/XlStHu6rdx6gd6doosIb94Kzwl2kmdqTd5jrn0siNauA+y8eyrSxurHwZMS1a0FMZBgb9/z3ouqBwlK+2pbDjDN6N3o9CneRZmjz3nyue24F+cXlPHf9mKCZodFLZsaAGhdVP0zZi6/CcYEH34I0n7tIM/PVtv1c+sRX+Cocr84cp2CvRwO7tWbTnkNUVF2Ufm/dXuLbt2CwB4t+K9xFmpGXlqZz9TMr6No6in/dOoHB3dt4XVJQGditNYWlPjIOFHGwqJSvUvdz/tBuntwHoG4ZkWagtLyCP76zgZeX7+SsAZ15dNqIJrkgdaAbWO2i6qHicso96pIBhbtI0MvILeKO19eyYnsuN5/Zh1/94JSgm3Y3UJzSJYYQg5Q9+XydeZC4di0YGuvNtyOFu0iQKi2v4Okv03js460Yxl8uH8FFI2O9LiuotYgIJaFjK5an5bBq5wGun9DLs6kZFO4iQaDcV8F/tmSTd7iMkvIKDpf6mL9iJ6lZBZw7uAuzfziY2FpmK5T6N7Bba977eg8A53t4r4DCXaSJKy7zcdsrq/kwZd93no9v34Jnr0vkrAFHzlIoDWdQVbjHtm3B8DjvLlgr3EWasIKScma+lMyS1Bx+f8FAzhnUhYiwECJCQ2jbMkJ96x4YWDW3+3lDuno6W6bCXaSJOlhUyrXPJbF+Vx4P/3g4l4yO87okAcYktGfSwC5cOa6np3Uo3EWaoN0HD3PtsyvYkVPE41eO4geDu3pdklSJiQrn6WsTvS5D4S7S1GzZl881z6ygsKSc538yhvF9OnpdkgQgv+5QNbPJZrbZzFLN7M6jtLnMzFLMbIOZza/fMkUEYMX2XC59/CsqnOPVmacp2OWo6jxzN7NQYC5wDpAJJJnZQudcSrU2/YC7gAnOuQNm1rmhChZpjsp9FbywdAcPfLCJuHYteOH6scS3b+l1WRLA/OmWGQukOufSAMxsATAVSKnW5kZgrnPuAIBzLqu+CxVprpLTc/n9W+vZtDefif078cjlI2jfKsLrsiTA+RPusUBGte1M4NQabfoDmNkSIBS4xzn3Qc0dmdkMYAZAjx6aO1rkWLLyi3nwg828sTKTbm2ieOKqUZw72NvhddJ0+BPutf0kuVr20w84E4gDvjCzIc65g995k3PzgHkAiYmJNfchIlROG/Dcku389ZNUSsp9zJzYm9vO6kerSI1/EP/589OSCcRX244DdtfSZplzrgzYbmabqQz7pHqpUqQZqKhwvL9+L//74Wa27y/k7AGd+d0FA+ndKdrr0qQJ8ifck4B+ZtYL2AVMA6bXaPMWcAXwvJl1pLKbJq0+CxUJVr4Kx7tf7+Zvn6SyNauAvp2jef76MZx5isYlyImrM9ydc+VmNgtYTGV/+rPOuQ1mNgdIds4trHrtB2aWAviAXzvnchqycJGm7Ju1Nb9M3c/nW7LZdfAwp3SJ4W/TR3LekG6aNkBOmjnnTdd3YmKiS05O9uSzRRpbdn4Jy9JySErPJSn9AJv2HsI5iIkM49TeHbhkVCznDu5KiEJd6mBmK51zdd4Cqys0Ig3staQMfv/2ekrLK2gZEcqoHu24fVJ/JvTtyPC4NoSFarVLqX8Kd5EGUlLu44/vpDB/+U4m9O3Ab84dwKDurQlXmEsjULiLNIDMA0XMmr+aNRkHuWliH371g/46Q5dGpXAXqUf7C0qY+2kqLy/bSXio8fiVozjPw9V4pPlSuIvUg8KSch7/bBvPLtlOSXkFl46K47ZJ/bS0nXhG4S5yEpyrvPHo3ndT2JNXzA+Hd+f2Sf1045F4TuEucoJSswr44zsb+GLrfgZ1a83fpo9idM92XpclAijcRY7b7oOHefSjrby+MoNWkWH8ccpgrhrXUzceSUBRuIv4qbjMx8MfbuaFpTvAwXXje3Hr9/vQITrS69JEjqBwF/FDXlEZN76YTNKOXC4dFcfPJ/Ujrp0Wy5DApXAXqcPevGKufXYFafsL+OsVI7lwWHevSxKpk8Jd5Bg27jnET19IJu9wGS9cP5bxfbVmqTQNCneRGsp8Ffw7ZR/zl+/ky9T9dIyOZMGMcQyJbeN1aSJ+U7iLVHHO8cbKTB5cvJns/BK6t4nijnP6M21sDzrF6KKpNC0KdxFgW3YBd7+5juXbcxndsx0PXDKUif07a3ijNFkKd2nWtu8v5JUVO3l+STpR4SHcf/FQLkuM17zq0uQp3KXZKS7z8c7a3byenMmK9FxCDKaOiOXu8weq+0WChl/hbmaTgUepXGbvaefc/TVevw54iMo1VgH+5px7uh7rFDlp+cVlvLx8J09/sZ39BSX07tiK304ewMWjYunSOsrr8kTqVZ3hbmahwFzgHCATSDKzhc65lBpNX3XOzWqAGkVOyuFSH0/8ZxvPLdnOoeJyTu/XkZsnjuC0Ph0wU/eLBCd/ztzHAqnOuTQAM1sATAVqhrtIQHHOsXjDXu59dyO7Dh7m3MFduPX7fRkW19br0kQanD/hHgtkVNvOBE6tpd0lZnYGsAW43TmXUbOBmc0AZgD06NHj+KsV8VPK7kPc/8EmPt+SzYCuMbw28zTG9mrvdVkijcafcK/te6ursf0O8IpzrsTMbgJeAM464k3OzQPmASQmJtbch8hJcc6xYnsuj/9nG59tziYmMozZFw7imtN6aok7aXb8CfdMIL7adhywu3oD51xOtc2ngAdOvjQR/63flcecd1JYkZ5Lh1YR/OoH/bl6XAJtWoZ7XZqIJ/wJ9ySgn5n1onI0zDRgevUGZtbNObenanMKsLFeqxQ5ikPFZfzfh1t4cWk67VtFMGfqYC5LjCcqPNTr0kQ8VWe4O+fKzWwWsJjKoZDPOuc2mNkcINk5txC4zcymAOVALnBdA9YsgnOOt9fs5r73NpJbWMJV43pyxw9OoU0LnamLAJhz3nR9JyYmuuTkZE8+W5q2tOwC/uft9SxJzWF4fFv+dNEQTeolzYaZrXTOJdbVTneoSpNxuNTH4//ZxhOfbSMyPIT7LhrCFWN7aP4XkVoo3CXgVVQ43l67iwc/2MyevGKmjujO7y4YSOcY3VUqcjQKdwloazMOMvvt9azNzGNobBsenTZS49VF/KBwl4BU5qvgr5+kMvfTVDq0iuDhHw/nRyNjNVujiJ8U7hJwUrMK+OVra/g6M4+LR8byhymDNQpG5Dgp3CWgfLB+D794dQ0twkN5/MpRnDe0m9cliTRJCncJCM45nvlyO39atJGR8W154urRumAqchIU7uI5X4Xj3ndTeP6rdM4b0pVHLh+hO0xFTpLCXTx1oLCU219bw2ebs7nx9F7cdd5AXTQVqQcKd/HMyh25zJq/mpyCUu67aAhXjevpdUkiQUPhLo3OV+F4+os0Hly8mdi2LfjnzeMZGqfpA0Tqk8JdGtXKHQe4Z+EG1u3K47whXXng0mG0jtIwR5H6pnCXRpF1qJj7P9jEm6t20bV1FI9OG8GU4d21hqlIA1G4S4P6Zmre2W+vp7isglu/34dbzuxLq0j96Ik0JP2GSYPJKSjh92+t5/31exndsx0PXTqM3p2ivS5LpFlQuEu9++9CGikcOlzOXecN4Ken99bUvCKNyK9Vg81sspltNrNUM7vzGO0uNTNnZnVOJC/BKTUrnyueWsYvXl1DbNsWvPOz7zFzYh8Fu0gjq/PM3cxCgbnAOVQulp1kZgudcyk12sUAtwHLG6JQCWzlvgoe+3grf/9sG60iw/jzj4YybUy8bkgS8Yg/3TJjgVTnXBqAmS0ApgIpNdrdCzwI/KpeK5SAl5VfzM/mr2b59lwuHhXL784fSIfoSK/LEmnW/An3WCCj2nYmcGr1BmY2Eoh3zr1rZgr3ZmR5Wg6zXllNfnEZ/3fZcC4eFed1SSKCf+Fe2/fqb1fVNrMQ4BHgujp3ZDYDmAHQo0cP/yqUgFTmq2Dup6n89ZNUerRvyUs3jGVA19ZelyUiVfwJ90wgvtp2HLC72nYMMAT4rOqGlK7AQjOb4pxLrr4j59w8YB5AYmKiQ5qkzXvzueP1NazfdYipI7pz30VDiNFdpiIBxZ9wTwL6mVkvYBcwDZj+zYvOuTyg4zfbZvYZ8KuawS5Nn6/CMe/zNB759xZiosJ44qpRTB6ixTREAlGd4e6cKzezWcBiIBR41jm3wczmAMnOuYUNXaR4Ly27gF+9vpZVOw9y3pCu3HfREF00FQlgft3E5JxbBCyq8dzso7Q98+TLkkBRUeF4cWk693+wiYjQEM0JI9JE6A5VqZVzjs+2ZPO/izezYfchzjylEw9cMowurbX0nUhToHCXIySn5/LAB5tISj9AfPsWPHL5cC4aEauzdZEmROEu38o7XMb972/klRUZdI6J5N6LhnB5YjwRYX7NUiEiAUThLgAs3rCX/3lrPfsLSph5Rm9+Mak/LSK0SLVIU6Vwb+ZW7zzAIx9t5fMt2Qzs1ppnrh2jJe9EgoDCvZlak3GQv3y0hc82Z9O+VQR3nz+A6yf0IjxUXTAiwUDh3sxkHijigQ82887a3bRrGc5vJw/gmtN6amUkkSCj3+hmorCknCf+s415n6cBcNtZfZkxsQ/RCnWRoKTf7Gbg8y3Z3PXmOnYdPMzUEd35zeQBxLZt4XVZItKAFO5BLK+ojPveS+H1lZn07tSK1286jTEJ7b0uS0QagcI9SH25dT+/fG0NOYWl3HJmH247ux9R4RraKNJcKNyDTJmvgoc/3MKTn2+jb6donr1uDENiNbRRpLlRuAeRnTlF3LZgNWsyDnLF2B7MvnCQbkQSaaYU7kHAOcfLy3fy50UbCQ0x5k4fxQXDNM+6SHOmcG/iMg8Ucec/1/Fl6n5O79eR+y8ZppEwIqJwb6qcc7yenMmcd1NwzvHnHw3lirHxmrlRRACFe5OUlV/MXf9cx8ebshjXuz0PXTqc+PYtvS5LRAKIX+FuZpOBR6lcZu9p59z9NV6/CbgV8AEFwAznXEo91yrAB+v3ctebX1NU6uN/LhzE9eMTCAnR2bqIfFed4W5mocBc4BwgE0gys4U1wnu+c+6JqvZTgP8DJjdAvc1WcZmPPy/ayItLdzA0tg2PXD6cvp1jvC5LRAKUP2fuY4FU51wagJktAKYC34a7c+5QtfatAFefRTZ36fsLuXX+KjbsPsRPv9eL30weoAU0ROSY/An3WCCj2nYmcGrNRmZ2K/BLIAI4q16qExat28Nv3via0BDjqWsSOWdQF69LEpEmwJ/Tv9o6dI84M3fOzXXO9QF+C/y+1h2ZzTCzZDNLzs7OPr5Km5kyXwX3vpvCLS+vom/naN677XsKdhHxmz9n7plAfLXtOGD3MdovAB6v7QXn3DxgHkBiYqK6bo5iT95hZs1fzcodB7hufAJ3nz9Q3TAiclz8CfckoJ+Z9QJ2AdOA6dUbmFk/59zWqs0LgK3ICfliazY/X7CGkjIff5s+kguHdfe6JBFpguoMd+dcuZnNAhZTORTyWefcBjObAyQ75xYCs8xsElAGHACubciig5GvwvHYx1t57JOt9O8cw9wrR9G3c7TXZYlIE+XXOHfn3CJgUY3nZld7/PN6rqtZySko4ecL1vBl6n4uGRXHfRcN0YRfInJSdIeqx9Zl5jHzpWRyCkt54JKhXJaoKQRE5OQp3D305qpM7npzHR2jI/nnzeM177qI1BuFuwfKfBX8edFGnluSzrje7Zk7fRQdoiO9LktEgojCvZFl55dw6/xVrNiey/UTEvjd+QMJC9UwRxGpXwr3RrR65wFu/scqDh4u5S+Xj+CikbFelyQiQUrh3kjmL9/JPQs30KVNJG/ePIFB3Vt7XZKIBDGFewMrLvNxz8INLEjK4Iz+nXhs2gjatozwuiwRCXIK9wa0++Bhbv7HStZm5jHr+325/Zz+hGrudRFpBAr3BpKcnsvMl1ZSUl7Bk1eP5tzBXb0uSUSaEYV7A3g9OYO7/7WOuHYteeqaRE0jICKNTuFej3wVjgc+2MS8z9OY0LcDc6ePUv+6iHhC4V5PCkrK+fkrq/l4UxZXj+vJ7B8OIlzj10XEIwr3epCRW8RPX0gmNbuAOVMHc81pCV6XJCLNnML9JK3ckcuMF1dS6qvg+evHcHq/Tl6XJCKicD8Z76zdzR2vraVb2yieuXaMLpyKSMBQuJ8A5xxPfp7G/e9vYkxCO+ZdnUi7VrpwKiKBQ+F+nMp9Fdzzzgb+sWwnFw7rxv/+eDhR4VpYQ0QCi1/DOcxsspltNrNUM7uzltd/aWYpZva1mX1sZj3rv1TvFZf5uPnlVfxj2U5mTuzNY9NGKthFJCDVGe5mFgrMBc4DBgFXmNmgGs1WA4nOuWHAG8CD9V2o1/KLy7j+uST+nbKPP04ZzF3nDSREUwmISIDy58x9LJDqnEtzzpUCC4Cp1Rs45z51zhVVbS4D4uq3TG/lFJQw/anlJKXn8ui0EVw7PsHrkkREjsmfcI8FMqptZ1Y9dzQ3AO/X9oKZzTCzZDNLzs7O9r9KD+3JO8xlTy5ly7585l0zmqkjNAe7iAQ+fy6o1tb34GptaHYVkAhMrO1159w8YB5AYmJirfsIJDtyCpn+1HLyDpfx4k/GcmrvDl6XJCLiF3/CPROIr7YdB+yu2cjMJgG/AyY650rqpzzvbNmXz1VPL6fMV8ErN45jaJwWrxaRpsOfbpkkoJ+Z9TKzCGAasLDcBJdwAAAHsklEQVR6AzMbCTwJTHHOZdV/mY1rXWYelz+5FIBXZ56mYBeRJqfOcHfOlQOzgMXARuA159wGM5tjZlOqmj0ERAOvm9kaM1t4lN0FvPW78rjy6WW0jAjj9ZtOo3+XGK9LEhE5bn7dxOScWwQsqvHc7GqPJ9VzXZ7YtPcQVz2znJiocBbMGEd8+5ZelyQickI0J22VrfvyufKp5USGhTD/xlMV7CLSpCncgfT9hUx/ejkhIcb8G8fRs0Mrr0sSETkpzT7c9+YVc+XTyyn3VTD/p6fSp5NmdhSRpq9ZTxyWW1jKVc9UjmN/5cZx9NPFUxEJEs023AtKyrn+uRXszC3ihevHarijiASVZtktU1pewcyXklm/+xB/nz6K0/rozlMRCS7NLtwrKhy/eWMtS1JzePCSYUwa1MXrkkRE6l2zC/eHPtzMW2t28+tzT+GS0UE1eaWIyLeaVbi/tDSdxz/bxpWn9uCWM/t4XY6ISINpNuH+8cZ9/GHhBiYN7MwfpwzGTAttiEjwahbhvnHPIW57ZTWDu7fhsStGEhbaLP7aItKMBX3KZeeX8NMXkomOCuOpaxJpGdFsR3+KSDMS1ElXXObjpn+sJKewhNdnjqdrmyivSxIRaRRBG+7OOe7+1zpW7jjA368cpZuURKRZCdpumee/SufNVbu4fVJ/zh/azetyREQaVVCG+/K0HO57byPnDOrCz87q63U5IiKNzq9wN7PJZrbZzFLN7M5aXj/DzFaZWbmZXVr/ZfpvT95hbp2/ip7tW/LwZcMJCdGQRxFpfuoMdzMLBeYC5wGDgCvMbFCNZjuB64D59V3g8Sgp93HzP1ZxuNTHvGtG0zoq3MtyREQ8488F1bFAqnMuDcDMFgBTgZRvGjjn0qteq2iAGv3inGP2WxtYk3GQJ64aRd/Omr5XRJovf7plYoGMatuZVc8FlBeX7uDV5Ax+dlZfJg/RBVQRad78CffaOq3diXyYmc0ws2QzS87Ozj6RXdRq6bYc5rybwqSBnbl9Uv9626+ISFPlT7hnAvHVtuOA3SfyYc65ec65ROdcYqdOnU5kF0fIyC3ilpdX0qtjKx65fIQuoIqI4F+4JwH9zKyXmUUA04CFDVuWf5al5XDNsyvwVTieuiaRGF1AFREB/Ah351w5MAtYDGwEXnPObTCzOWY2BcDMxphZJvBj4Ekz29CQRe86WDnccdq8ZZSWV/DUNYn06tiqIT9SRKRJ8Wv6AefcImBRjedmV3ucRGV3TYN7LSmD2QvXA3D7pP7MnNibqPDQxvhoEZEmo8nNLdOzQ0vOHtCFuy8YSGzbFl6XIyISkJpcuJ/auwOn9taC1iIixxKUc8uIiDR3CncRkSCkcBcRCUIKdxGRIKRwFxEJQgp3EZEgpHAXEQlCCncRkSBkzp3Q7L0n/8Fm2cCOE3x7R2B/PZYTLHRcaqfjcnQ6NrUL5OPS0zlX57S6noX7yTCzZOdcotd1BBodl9rpuBydjk3tguG4qFtGRCQIKdxFRIJQUw33eV4XEKB0XGqn43J0Oja1a/LHpUn2uYuIyLE11TN3ERE5hoAOdzObbGabzSzVzO6s5fVIM3u16vXlZpbQ+FU2Pj+Oyy/NLMXMvjazj82spxd1Nra6jku1dpeamTOzJj0awl/+HBczu6zqZ2aDmc1v7Bq94MfvUQ8z+9TMVlf9Lp3vRZ0nzDkXkH+AUGAb0BuIANYCg2q0uQV4ourxNOBVr+sOkOPyfaBl1eObdVy+0y4G+BxYBiR6XXcgHBegH7AaaFe13dnrugPkuMwDbq56PAhI97ru4/kTyGfuY4FU51yac64UWABMrdFmKvBC1eM3gLPNzBqxRi/UeVycc58654qqNpfRSOvbesyfnxeAe4EHgeLGLM5D/hyXG4G5zrkDAM65rEau0Qv+HBcHtK563AbY3Yj1nbRADvdYIKPadmbVc7W2cc6VA3lAsK/B589xqe4G4P0GrSgw1HlczGwkEO+ce7cxC/OYPz8v/YH+ZrbEzJaZ2eRGq847/hyXe4CrzCwTWAT8rHFKqx+BvIZqbWfgNYf2+NMm2Pj9dzazq4BEYGKDVhQYjnlczCwEeAS4rrEKChD+/LyEUdk1cyaV3/K+MLMhzrmDDVybl/w5LlcAzzvnHjaz04CXqo5LRcOXd/IC+cw9E4ivth3HkV+Lvm1jZmFUfnXKbZTqvOPPccHMJgG/A6Y450oaqTYv1XVcYoAhwGdmlg6MAxY2g4uq/v4eve2cK3PObQc2Uxn2wcyf43ID8BqAc24pEEXlnDNNQiCHexLQz8x6mVkElRdMF9ZosxC4turxpcAnrurqRxCr87hUdT88SWWwN4f+U6jjuDjn8pxzHZ1zCc65BCqvRUxxziV7U26j8ef36C0qL8JjZh2p7KZJa9QqG58/x2UncDaAmQ2kMtyzG7XKkxCw4V7Vhz4LWAxsBF5zzm0wszlmNqWq2TNABzNLBX4JHHX4W7Dw87g8BEQDr5vZGjOr+UMbdPw8Ls2On8dlMZBjZinAp8CvnXM53lTcOPw8LncAN5rZWuAV4LqmdPKoO1RFRIJQwJ65i4jIiVO4i4gEIYW7iEgQUriLiAQhhbuISBBSuIuIBCGFu4hIEFK4i4gEof8PmoPsBTlnF6YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f17894aa898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X = []\n",
    "Y = []\n",
    "for i in np.linspace(0,1,100)[::-1]:\n",
    "    df_ = df[df.y_pred>i]\n",
    "    if df_.shape[0]>3:\n",
    "        X.append(i)\n",
    "        Y.append(df_.y.mean())\n",
    "plt.plot(X, Y); plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T08:28:31.850752Z",
     "start_time": "2018-08-04T08:28:31.835848Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((234942, 2), 0.037524154897804564)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.y_pred<0.1].shape, df[df.y_pred<0.1].y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T08:29:02.215742Z",
     "start_time": "2018-08-04T08:29:02.205950Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((29318, 2), 0.005764376833344703)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.y_pred<0.01].shape, df[df.y_pred<0.01].y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:11:59.118919Z",
     "start_time": "2018-08-04T14:11:59.116480Z"
    }
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:12:37.597447Z",
     "start_time": "2018-08-04T14:12:36.006917Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(df[['y_pred']], df.y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:15:57.435738Z",
     "start_time": "2018-08-04T14:15:57.341208Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test = pd.read_csv('../data/LB804_test_pred.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:16:04.256936Z",
     "start_time": "2018-08-04T14:16:04.201507Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.035935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.111796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.018036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.047114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.139933</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     y_pred\n",
       "0  0.035935\n",
       "1  0.111796\n",
       "2  0.018036\n",
       "3  0.047114\n",
       "4  0.139933"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:17:26.131399Z",
     "start_time": "2018-08-04T14:17:26.126295Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test['y_pred_lr'] = lr.predict_proba(X_test)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:17:30.680359Z",
     "start_time": "2018-08-04T14:17:30.666232Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>y_pred</th>\n",
       "      <th>y_pred_lr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.035935</td>\n",
       "      <td>0.045083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.111796</td>\n",
       "      <td>0.079500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.018036</td>\n",
       "      <td>0.039331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.047114</td>\n",
       "      <td>0.049074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.139933</td>\n",
       "      <td>0.097514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.026047</td>\n",
       "      <td>0.041813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.006335</td>\n",
       "      <td>0.035959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.043883</td>\n",
       "      <td>0.047887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.009755</td>\n",
       "      <td>0.036914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.073341</td>\n",
       "      <td>0.059787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.033017</td>\n",
       "      <td>0.044094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.024367</td>\n",
       "      <td>0.041281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.225128</td>\n",
       "      <td>0.175536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.051552</td>\n",
       "      <td>0.050749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.059744</td>\n",
       "      <td>0.053985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.187564</td>\n",
       "      <td>0.136349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.030534</td>\n",
       "      <td>0.043268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.016612</td>\n",
       "      <td>0.038905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.116987</td>\n",
       "      <td>0.082577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.015689</td>\n",
       "      <td>0.038631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.042814</td>\n",
       "      <td>0.047501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.011568</td>\n",
       "      <td>0.037431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.043491</td>\n",
       "      <td>0.047745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.264189</td>\n",
       "      <td>0.225148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.052082</td>\n",
       "      <td>0.050953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.116874</td>\n",
       "      <td>0.082509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.082065</td>\n",
       "      <td>0.063813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.091707</td>\n",
       "      <td>0.068556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.019182</td>\n",
       "      <td>0.039677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.045962</td>\n",
       "      <td>0.048648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48714</th>\n",
       "      <td>0.023954</td>\n",
       "      <td>0.041151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48715</th>\n",
       "      <td>0.006826</td>\n",
       "      <td>0.036094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48716</th>\n",
       "      <td>0.295057</td>\n",
       "      <td>0.270879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48717</th>\n",
       "      <td>0.020697</td>\n",
       "      <td>0.040139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48718</th>\n",
       "      <td>0.064718</td>\n",
       "      <td>0.056043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48719</th>\n",
       "      <td>0.073493</td>\n",
       "      <td>0.059855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48720</th>\n",
       "      <td>0.018077</td>\n",
       "      <td>0.039343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48721</th>\n",
       "      <td>0.133391</td>\n",
       "      <td>0.093026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48722</th>\n",
       "      <td>0.191702</td>\n",
       "      <td>0.140275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48723</th>\n",
       "      <td>0.110443</td>\n",
       "      <td>0.078715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48724</th>\n",
       "      <td>0.055543</td>\n",
       "      <td>0.052302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48725</th>\n",
       "      <td>0.256351</td>\n",
       "      <td>0.214448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48726</th>\n",
       "      <td>0.018735</td>\n",
       "      <td>0.039542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48727</th>\n",
       "      <td>0.007031</td>\n",
       "      <td>0.036151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48728</th>\n",
       "      <td>0.006603</td>\n",
       "      <td>0.036033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48729</th>\n",
       "      <td>0.095899</td>\n",
       "      <td>0.070717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48730</th>\n",
       "      <td>0.075910</td>\n",
       "      <td>0.060947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48731</th>\n",
       "      <td>0.007859</td>\n",
       "      <td>0.036382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48732</th>\n",
       "      <td>0.152312</td>\n",
       "      <td>0.106538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48733</th>\n",
       "      <td>0.025742</td>\n",
       "      <td>0.041716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48734</th>\n",
       "      <td>0.035054</td>\n",
       "      <td>0.044782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48735</th>\n",
       "      <td>0.056647</td>\n",
       "      <td>0.052739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48736</th>\n",
       "      <td>0.011942</td>\n",
       "      <td>0.037539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48737</th>\n",
       "      <td>0.119965</td>\n",
       "      <td>0.084391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48738</th>\n",
       "      <td>0.057447</td>\n",
       "      <td>0.053058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48739</th>\n",
       "      <td>0.057709</td>\n",
       "      <td>0.053163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48740</th>\n",
       "      <td>0.054124</td>\n",
       "      <td>0.051745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48741</th>\n",
       "      <td>0.005213</td>\n",
       "      <td>0.035650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48742</th>\n",
       "      <td>0.018196</td>\n",
       "      <td>0.039379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48743</th>\n",
       "      <td>0.090355</td>\n",
       "      <td>0.067871</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>48744 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         y_pred  y_pred_lr\n",
       "0      0.035935   0.045083\n",
       "1      0.111796   0.079500\n",
       "2      0.018036   0.039331\n",
       "3      0.047114   0.049074\n",
       "4      0.139933   0.097514\n",
       "5      0.026047   0.041813\n",
       "6      0.006335   0.035959\n",
       "7      0.043883   0.047887\n",
       "8      0.009755   0.036914\n",
       "9      0.073341   0.059787\n",
       "10     0.033017   0.044094\n",
       "11     0.024367   0.041281\n",
       "12     0.225128   0.175536\n",
       "13     0.051552   0.050749\n",
       "14     0.059744   0.053985\n",
       "15     0.187564   0.136349\n",
       "16     0.030534   0.043268\n",
       "17     0.016612   0.038905\n",
       "18     0.116987   0.082577\n",
       "19     0.015689   0.038631\n",
       "20     0.042814   0.047501\n",
       "21     0.011568   0.037431\n",
       "22     0.043491   0.047745\n",
       "23     0.264189   0.225148\n",
       "24     0.052082   0.050953\n",
       "25     0.116874   0.082509\n",
       "26     0.082065   0.063813\n",
       "27     0.091707   0.068556\n",
       "28     0.019182   0.039677\n",
       "29     0.045962   0.048648\n",
       "...         ...        ...\n",
       "48714  0.023954   0.041151\n",
       "48715  0.006826   0.036094\n",
       "48716  0.295057   0.270879\n",
       "48717  0.020697   0.040139\n",
       "48718  0.064718   0.056043\n",
       "48719  0.073493   0.059855\n",
       "48720  0.018077   0.039343\n",
       "48721  0.133391   0.093026\n",
       "48722  0.191702   0.140275\n",
       "48723  0.110443   0.078715\n",
       "48724  0.055543   0.052302\n",
       "48725  0.256351   0.214448\n",
       "48726  0.018735   0.039542\n",
       "48727  0.007031   0.036151\n",
       "48728  0.006603   0.036033\n",
       "48729  0.095899   0.070717\n",
       "48730  0.075910   0.060947\n",
       "48731  0.007859   0.036382\n",
       "48732  0.152312   0.106538\n",
       "48733  0.025742   0.041716\n",
       "48734  0.035054   0.044782\n",
       "48735  0.056647   0.052739\n",
       "48736  0.011942   0.037539\n",
       "48737  0.119965   0.084391\n",
       "48738  0.057447   0.053058\n",
       "48739  0.057709   0.053163\n",
       "48740  0.054124   0.051745\n",
       "48741  0.005213   0.035650\n",
       "48742  0.018196   0.039379\n",
       "48743  0.090355   0.067871\n",
       "\n",
       "[48744 rows x 2 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:18:55.351400Z",
     "start_time": "2018-08-04T14:18:55.156281Z"
    }
   },
   "outputs": [],
   "source": [
    "X_test.to_csv('../data/LB804_test_pred2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:27:45.928765Z",
     "start_time": "2018-08-04T14:27:45.896519Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    48744.000000\n",
       "mean         0.069770\n",
       "std          0.077432\n",
       "min          0.034541\n",
       "25%          0.038905\n",
       "50%          0.045087\n",
       "75%          0.063688\n",
       "max          0.951885\n",
       "Name: y_pred_lr, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.y_pred_lr.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-04T14:43:19.592662Z",
     "start_time": "2018-08-04T14:43:19.586323Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.036638996706195644"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.y_pred_lr.quantile(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
